src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "http/Stream.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47 #if USE_SQUID_ESI
  48 #include "esi/Esi.h"
  49 #endif
  50
  51 #include <memory>
  52
  53 CBDATA_CLASS_INIT(clientReplyContext);
  54
  55 /* Local functions */
  56 extern "C" CSS clientReplyStatus;
  57 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  58
  59 /* privates */
  60
  61 clientReplyContext::~clientReplyContext()
  62 {
  63     deleting = true;
  64     /* This may trigger a callback back into SendMoreData as the cbdata
  65      * is still valid
  66      */
  67     removeClientStoreReference(&sc, http);
  68     /* old_entry might still be set if we didn't yet get the reply
  69      * code in HandleIMSReply() */
  70     removeStoreReference(&old_sc, &old_entry);
  71     safe_free(tempBuffer.data);
  72     cbdataReferenceDone(http);
  73     HTTPMSGUNLOCK(reply);
  74 }
  75
  76 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  77     purgeStatus(Http::scNone),
  78     lookingforstore(0),
  79     http(cbdataReference(clientContext)),
  80     headers_sz(0),
  81     sc(NULL),
  82     old_reqsize(0),
  83     reqsize(0),
  84     reqofs(0),
  85 #if USE_CACHE_DIGESTS
  86     lookup_type(NULL),
  87 #endif
  88     ourNode(NULL),
  89     reply(NULL),
  90     old_entry(NULL),
  91     old_sc(NULL),
  92     deleting(false),
  93     collapsedRevalidation(crNone)
  94 {
  95     *tempbuf = 0;
  96 }
  97
  98 /** Create an error in the store awaiting the client side to read it.
  99  *
 100  * This may be better placed in the clientStream logic, but it has not been
 101  * relocated there yet
 102  */
 103 void
 104 clientReplyContext::setReplyToError(
 105     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 106     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 107 #if USE_AUTH
 108     Auth::UserRequest::Pointer auth_user_request
 109 #else
 110     void*
 111 #endif
 112 )
 113 {
 114     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 115
 116     if (unparsedrequest)
 117         errstate->request_hdrs = xstrdup(unparsedrequest);
 118
 119 #if USE_AUTH
 120     errstate->auth_user_request = auth_user_request;
 121 #endif
 122     setReplyToError(method, errstate);
 123 }
 124
 125 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 126 {
 127     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 128         /* prevent confusion over whether we default to persistent or not */
 129         http->request->flags.proxyKeepalive = false;
 130
 131     http->al->http.code = errstate->httpStatus;
 132
 133     if (http->request)
 134         http->request->ignoreRange("responding with a Squid-generated error");
 135
 136     createStoreEntry(method, RequestFlags());
 137     assert(errstate->callback_data == NULL);
 138     errorAppendEntry(http->storeEntry(), errstate);
 139     /* Now the caller reads to get this */
 140 }
 141
 142 void
 143 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 144 {
 145     Must(futureReply);
 146     http->al->http.code = futureReply->sline.status();
 147
 148     HttpRequestMethod method;
 149     if (http->request) { // nil on responses to unparsable requests
 150         http->request->ignoreRange("responding with a Squid-generated reply");
 151         method = http->request->method;
 152     }
 153
 154     createStoreEntry(method, RequestFlags());
 155
 156     http->storeEntry()->storeErrorResponse(futureReply);
 157     /* Now the caller reads to get futureReply */
 158 }
 159
 160 // Assumes that the entry contains an error response without Content-Range.
 161 // To use with regular entries, make HTTP Range header removal conditional.
 162 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 163 {
 164     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 165     sc = storeClientListAdd(entry, this);
 166 #if USE_DELAY_POOLS
 167     sc->setDelayId(DelayId::DelayClient(http));
 168 #endif
 169     reqofs = 0;
 170     reqsize = 0;
 171     if (http->request)
 172         http->request->ignoreRange(reason);
 173     flags.storelogiccomplete = 1;
 174     http->storeEntry(entry);
 175 }
 176
 177 void
 178 clientReplyContext::removeStoreReference(store_client ** scp,
 179         StoreEntry ** ep)
 180 {
 181     StoreEntry *e;
 182     store_client *sc_tmp = *scp;
 183
 184     if ((e = *ep) != NULL) {
 185         *ep = NULL;
 186         storeUnregister(sc_tmp, e, this);
 187         *scp = NULL;
 188         e->unlock("clientReplyContext::removeStoreReference");
 189     }
 190 }
 191
 192 void
 193 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 194 {
 195     StoreEntry *reference = aHttpRequest->storeEntry();
 196     removeStoreReference(scp, &reference);
 197     aHttpRequest->storeEntry(reference);
 198 }
 199
 200 void
 201 clientReplyContext::saveState()
 202 {
 203     assert(old_sc == NULL);
 204     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 205     old_entry = http->storeEntry();
 206     old_sc = sc;
 207     old_reqsize = reqsize;
 208     tempBuffer.offset = reqofs;
 209     /* Prevent accessing the now saved entries */
 210     http->storeEntry(NULL);
 211     sc = NULL;
 212     reqsize = 0;
 213     reqofs = 0;
 214 }
 215
 216 void
 217 clientReplyContext::restoreState()
 218 {
 219     assert(old_sc != NULL);
 220     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 221     removeClientStoreReference(&sc, http);
 222     http->storeEntry(old_entry);
 223     sc = old_sc;
 224     reqsize = old_reqsize;
 225     reqofs = tempBuffer.offset;
 226     /* Prevent accessed the old saved entries */
 227     old_entry = NULL;
 228     old_sc = NULL;
 229     old_reqsize = 0;
 230     tempBuffer.offset = 0;
 231 }
 232
 233 void
 234 clientReplyContext::startError(ErrorState * err)
 235 {
 236     createStoreEntry(http->request->method, RequestFlags());
 237     triggerInitialStoreRead();
 238     errorAppendEntry(http->storeEntry(), err);
 239 }
 240
 241 clientStreamNode *
 242 clientReplyContext::getNextNode() const
 243 {
 244     return (clientStreamNode *)ourNode->node.next->data;
 245 }
 246
 247 /* This function is wrong - the client parameters don't include the
 248  * header offset
 249  */
 250 void
 251 clientReplyContext::triggerInitialStoreRead()
 252 {
 253     /* when confident, 0 becomes reqofs, and then this factors into
 254      * startSendProcess
 255      */
 256     assert(reqofs == 0);
 257     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 258     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 259 }
 260
 261 /* there is an expired entry in the store.
 262  * setup a temporary buffer area and perform an IMS to the origin
 263  */
 264 void
 265 clientReplyContext::processExpired()
 266 {
 267     const char *url = storeId();
 268     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 269     assert(http->storeEntry()->lastmod >= 0);
 270     /*
 271      * check if we are allowed to contact other servers
 272      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 273      *      a stale entry *if* it matches client requirements
 274      */
 275
 276     if (http->onlyIfCached()) {
 277         processOnlyIfCachedMiss();
 278         return;
 279     }
 280
 281     http->logType = LOG_TCP_REFRESH;
 282     http->request->flags.refresh = true;
 283 #if STORE_CLIENT_LIST_DEBUG
 284     /* Prevent a race with the store client memory free routines
 285      */
 286     assert(storeClientIsThisAClient(sc, this));
 287 #endif
 288     /* Prepare to make a new temporary request */
 289     saveState();
 290
 291     // TODO: support collapsed revalidation for Vary-controlled entries
 292     const bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
 293                                    !Store::Root().smpAware() &&
 294                                    http->request->vary_headers.isEmpty();
 295
 296     StoreEntry *entry = nullptr;
 297     if (collapsingAllowed) {
 298         if ((entry = storeGetPublicByRequest(http->request, ksRevalidation)))
 299             entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
 300     }
 301
 302     if (entry) {
 303         debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
 304         collapsedRevalidation = crSlave;
 305     } else {
 306         entry = storeCreateEntry(url,
 307                                  http->log_uri, http->request->flags, http->request->method);
 308         /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 309
 310         if (collapsingAllowed) {
 311             debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
 312             Store::Root().allowCollapsing(entry, http->request->flags,
 313                                           http->request->method);
 314             collapsedRevalidation = crInitiator;
 315         } else {
 316             collapsedRevalidation = crNone;
 317         }
 318     }
 319
 320     sc = storeClientListAdd(entry, this);
 321 #if USE_DELAY_POOLS
 322     /* delay_id is already set on original store client */
 323     sc->setDelayId(DelayId::DelayClient(http));
 324 #endif
 325
 326     http->request->lastmod = old_entry->lastmod;
 327
 328     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 329         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 330         if (old_entry->hasEtag(etag) && !etag.weak)
 331             http->request->etag = etag.str;
 332     }
 333
 334     debugs(88, 5, "clientReplyContext::processExpired : lastmod " << entry->lastmod );
 335     http->storeEntry(entry);
 336     assert(http->out.offset == 0);
 337     assert(http->request->clientConnectionManager == http->getConn());
 338
 339     if (collapsedRevalidation != crSlave) {
 340         /*
 341          * A refcounted pointer so that FwdState stays around as long as
 342          * this clientReplyContext does
 343          */
 344         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 345         FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 346     }
 347     /* Register with storage manager to receive updates when data comes in. */
 348
 349     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 350         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 351
 352     {
 353         /* start counting the length from 0 */
 354         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 355         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 356     }
 357 }
 358
 359 void
 360 clientReplyContext::sendClientUpstreamResponse()
 361 {
 362     StoreIOBuffer tempresult;
 363     removeStoreReference(&old_sc, &old_entry);
 364     /* here the data to send is the data we just received */
 365     tempBuffer.offset = 0;
 366     old_reqsize = 0;
 367     /* sendMoreData tracks the offset as well.
 368      * Force it back to zero */
 369     reqofs = 0;
 370     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 371     /* TODO: provide sendMoreData with the ready parsed reply */
 372     tempresult.length = reqsize;
 373     tempresult.data = tempbuf;
 374     sendMoreData(tempresult);
 375 }
 376
 377 void
 378 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 379 {
 380     clientReplyContext *context = (clientReplyContext *)data;
 381     context->handleIMSReply(result);
 382 }
 383
 384 void
 385 clientReplyContext::sendClientOldEntry()
 386 {
 387     /* Get the old request back */
 388     restoreState();
 389     /* here the data to send is in the next nodes buffers already */
 390     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 391     /* sendMoreData tracks the offset as well.
 392      * Force it back to zero */
 393     reqofs = 0;
 394     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 395     sendMoreData(tempresult);
 396 }
 397
 398 /* This is the workhorse of the HandleIMSReply callback.
 399  *
 400  * It is called when we've got data back from the origin following our
 401  * IMS request to revalidate a stale entry.
 402  */
 403 void
 404 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 405 {
 406     if (deleting)
 407         return;
 408
 409     debugs(88, 3, "handleIMSReply: " << http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes" );
 410
 411     if (http->storeEntry() == NULL)
 412         return;
 413
 414     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 415         return;
 416
 417     /* update size of the request */
 418     reqsize = result.length + reqofs;
 419
 420     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 421
 422     // request to origin was aborted
 423     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 424         debugs(88, 3, "handleIMSReply: request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client" );
 425         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 426         sendClientOldEntry();
 427     }
 428
 429     const HttpReply *old_rep = old_entry->getReply();
 430
 431     // origin replied 304
 432     if (status == Http::scNotModified) {
 433         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 434         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 435
 436         // update headers on existing entry
 437         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 438
 439         // if client sent IMS
 440
 441         if (http->request->flags.ims && !old_entry->modifiedSince(http->request)) {
 442             // forward the 304 from origin
 443             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client");
 444             sendClientUpstreamResponse();
 445         } else {
 446             // send existing entry, it's still valid
 447             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " <<
 448                    old_rep->sline.status() << " to client");
 449             sendClientOldEntry();
 450         }
 451     }
 452
 453     // origin replied with a non-error code
 454     else if (status > Http::scNone && status < Http::scInternalServerError) {
 455         // forward response from origin
 456         http->logType = LOG_TCP_REFRESH_MODIFIED;
 457         debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client");
 458
 459         if (collapsedRevalidation)
 460             http->storeEntry()->clearPublicKeyScope();
 461
 462         sendClientUpstreamResponse();
 463     }
 464
 465     // origin replied with an error
 466     else if (http->request->flags.failOnValidationError) {
 467         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 468         debugs(88, 3, "handleIMSReply: origin replied with error " << status <<
 469                ", forwarding to client due to fail_on_validation_err");
 470         sendClientUpstreamResponse();
 471     } else {
 472         // ignore and let client have old entry
 473         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 474         debugs(88, 3, "handleIMSReply: origin replied with error " <<
 475                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 476         sendClientOldEntry();
 477     }
 478 }
 479
 480 SQUIDCEXTERN CSR clientGetMoreData;
 481 SQUIDCEXTERN CSD clientReplyDetach;
 482
 483 /**
 484  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 485  * have been parsed.  Normally this should be a single call, but
 486  * it might take more than one.  As soon as we have the headers,
 487  * we hand off to clientSendMoreData, processExpired, or
 488  * processMiss.
 489  */
 490 void
 491 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 492 {
 493     clientReplyContext *context = (clientReplyContext *)data;
 494     context->cacheHit(result);
 495 }
 496
 497 /**
 498  * Process a possible cache HIT.
 499  */
 500 void
 501 clientReplyContext::cacheHit(StoreIOBuffer result)
 502 {
 503     /** Ignore if the HIT object is being deleted. */
 504     if (deleting) {
 505         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 506         return;
 507     }
 508
 509     StoreEntry *e = http->storeEntry();
 510
 511     HttpRequest *r = http->request;
 512
 513     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 514
 515     if (http->storeEntry() == NULL) {
 516         debugs(88, 3, "clientCacheHit: request aborted");
 517         return;
 518     } else if (result.flags.error) {
 519         /* swap in failure */
 520         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 521         http->logType = LOG_TCP_SWAPFAIL_MISS;
 522         removeClientStoreReference(&sc, http);
 523         processMiss();
 524         return;
 525     }
 526
 527     if (result.length == 0) {
 528         debugs(88, 5, "store IO buffer has no content. MISS");
 529         /* the store couldn't get enough data from the file for us to id the
 530          * object
 531          */
 532         /* treat as a miss */
 533         http->logType = LOG_TCP_MISS;
 534         processMiss();
 535         return;
 536     }
 537
 538     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 539     /* update size of the request */
 540     reqsize = result.length + reqofs;
 541
 542     /*
 543      * Got the headers, now grok them
 544      */
 545     assert(http->logType.oldType == LOG_TCP_HIT);
 546
 547     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 548         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 549         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 550         processMiss();
 551         return;
 552     }
 553
 554     switch (varyEvaluateMatch(e, r)) {
 555
 556     case VARY_NONE:
 557         /* No variance detected. Continue as normal */
 558         break;
 559
 560     case VARY_MATCH:
 561         /* This is the correct entity for this request. Continue */
 562         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 563         break;
 564
 565     case VARY_OTHER:
 566         /* This is not the correct entity for this request. We need
 567          * to requery the cache.
 568          */
 569         removeClientStoreReference(&sc, http);
 570         e = NULL;
 571         /* Note: varyEvalyateMatch updates the request with vary information
 572          * so we only get here once. (it also takes care of cancelling loops)
 573          */
 574         debugs(88, 2, "clientProcessHit: Vary detected!");
 575         clientGetMoreData(ourNode, http);
 576         return;
 577
 578     case VARY_CANCEL:
 579         /* varyEvaluateMatch found a object loop. Process as miss */
 580         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 581         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 582         processMiss();
 583         return;
 584     }
 585
 586     if (r->method == Http::METHOD_PURGE) {
 587         debugs(88, 5, "PURGE gets a HIT");
 588         removeClientStoreReference(&sc, http);
 589         e = NULL;
 590         purgeRequest();
 591         return;
 592     }
 593
 594     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 595         debugs(88, 5, "negative-HIT");
 596         http->logType = LOG_TCP_NEGATIVE_HIT;
 597         sendMoreData(result);
 598     } else if (blockedHit()) {
 599         debugs(88, 5, "send_hit forces a MISS");
 600         http->logType = LOG_TCP_MISS;
 601         processMiss();
 602         return;
 603     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 604         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 605         /*
 606          * We hold a stale copy; it needs to be validated
 607          */
 608         /*
 609          * The 'needValidation' flag is used to prevent forwarding
 610          * loops between siblings.  If our copy of the object is stale,
 611          * then we should probably only use parents for the validation
 612          * request.  Otherwise two siblings could generate a loop if
 613          * both have a stale version of the object.
 614          */
 615         r->flags.needValidation = true;
 616
 617         if (e->lastmod < 0) {
 618             debugs(88, 3, "validate HIT object? NO. Missing Last-Modified header. Do MISS.");
 619             /*
 620              * Previous reply didn't have a Last-Modified header,
 621              * we cannot revalidate it.
 622              */
 623             http->logType = LOG_TCP_MISS;
 624             processMiss();
 625         } else if (r->flags.noCache) {
 626             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 627             /*
 628              * This did not match a refresh pattern that overrides no-cache
 629              * we should honour the client no-cache header.
 630              */
 631             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 632             processMiss();
 633         } else if (r->url.getScheme() == AnyP::PROTO_HTTP) {
 634             debugs(88, 3, "validate HIT object? YES.");
 635             /*
 636              * Object needs to be revalidated
 637              * XXX This could apply to FTP as well, if Last-Modified is known.
 638              */
 639             processExpired();
 640         } else {
 641             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 642             /*
 643              * We don't know how to re-validate other protocols. Handle
 644              * them as if the object has expired.
 645              */
 646             http->logType = LOG_TCP_MISS;
 647             processMiss();
 648         }
 649     } else if (r->conditional()) {
 650         debugs(88, 5, "conditional HIT");
 651         processConditional(result);
 652     } else {
 653         /*
 654          * plain ol' cache hit
 655          */
 656         debugs(88, 5, "plain old HIT");
 657
 658 #if USE_DELAY_POOLS
 659         if (e->store_status != STORE_OK)
 660             http->logType = LOG_TCP_MISS;
 661         else
 662 #endif
 663             if (e->mem_status == IN_MEMORY)
 664                 http->logType = LOG_TCP_MEM_HIT;
 665             else if (Config.onoff.offline)
 666                 http->logType = LOG_TCP_OFFLINE_HIT;
 667
 668         sendMoreData(result);
 669     }
 670 }
 671
 672 /**
 673  * Prepare to fetch the object as it's a cache miss of some kind.
 674  */
 675 void
 676 clientReplyContext::processMiss()
 677 {
 678     char *url = http->uri;
 679     HttpRequest *r = http->request;
 680     ErrorState *err = NULL;
 681     debugs(88, 4, r->method << ' ' << url);
 682
 683     /**
 684      * We might have a left-over StoreEntry from a failed cache hit
 685      * or IMS request.
 686      */
 687     if (http->storeEntry()) {
 688         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 689             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 690             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 691             http->storeEntry()->dump(1);
 692         }
 693
 694         removeClientStoreReference(&sc, http);
 695     }
 696
 697     /** Check if its a PURGE request to be actioned. */
 698     if (r->method == Http::METHOD_PURGE) {
 699         purgeRequest();
 700         return;
 701     }
 702
 703     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 704     if (r->method == Http::METHOD_OTHER) {
 705         purgeAllCached();
 706     }
 707
 708     /** Check if 'only-if-cached' flag is set. Action if so. */
 709     if (http->onlyIfCached()) {
 710         processOnlyIfCachedMiss();
 711         return;
 712     }
 713
 714     /// Deny loops
 715     if (r->flags.loopDetected) {
 716         http->al->http.code = Http::scForbidden;
 717         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 718         createStoreEntry(r->method, RequestFlags());
 719         errorAppendEntry(http->storeEntry(), err);
 720         triggerInitialStoreRead();
 721         return;
 722     } else {
 723         assert(http->out.offset == 0);
 724         createStoreEntry(r->method, r->flags);
 725         triggerInitialStoreRead();
 726
 727         if (http->redirect.status) {
 728             HttpReply *rep = new HttpReply;
 729             http->logType = LOG_TCP_REDIRECT;
 730             http->storeEntry()->releaseRequest();
 731             rep->redirect(http->redirect.status, http->redirect.location);
 732             http->storeEntry()->replaceHttpReply(rep);
 733             http->storeEntry()->complete();
 734             return;
 735         }
 736
 737         assert(r->clientConnectionManager == http->getConn());
 738
 739         /** Start forwarding to get the new object from network */
 740         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 741         FwdState::Start(conn, http->storeEntry(), r, http->al);
 742     }
 743 }
 744
 745 /**
 746  * client issued a request with an only-if-cached cache-control directive;
 747  * we did not find a cached object that can be returned without
 748  *     contacting other servers;
 749  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 750  */
 751 void
 752 clientReplyContext::processOnlyIfCachedMiss()
 753 {
 754     debugs(88, 4, http->request->method << ' ' << http->uri);
 755     http->al->http.code = Http::scGatewayTimeout;
 756     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 757                                        http->getConn()->clientConnection->remote, http->request);
 758     removeClientStoreReference(&sc, http);
 759     startError(err);
 760 }
 761
 762 /// process conditional request from client
 763 void
 764 clientReplyContext::processConditional(StoreIOBuffer &result)
 765 {
 766     StoreEntry *const e = http->storeEntry();
 767
 768     if (e->getReply()->sline.status() != Http::scOkay) {
 769         debugs(88, 4, "clientReplyContext::processConditional: Reply code " <<
 770                e->getReply()->sline.status() << " != 200");
 771         http->logType = LOG_TCP_MISS;
 772         processMiss();
 773         return;
 774     }
 775
 776     HttpRequest &r = *http->request;
 777
 778     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 779         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 780         sendPreconditionFailedError();
 781         return;
 782     }
 783
 784     bool matchedIfNoneMatch = false;
 785     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 786         if (!e->hasIfNoneMatchEtag(r)) {
 787             // RFC 2616: ignore IMS if If-None-Match did not match
 788             r.flags.ims = false;
 789             r.ims = -1;
 790             r.imslen = 0;
 791             r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 792             http->logType = LOG_TCP_MISS;
 793             sendMoreData(result);
 794             return;
 795         }
 796
 797         if (!r.flags.ims) {
 798             // RFC 2616: if If-None-Match matched and there is no IMS,
 799             // reply with 304 Not Modified or 412 Precondition Failed
 800             sendNotModifiedOrPreconditionFailedError();
 801             return;
 802         }
 803
 804         // otherwise check IMS below to decide if we reply with 304 or 412
 805         matchedIfNoneMatch = true;
 806     }
 807
 808     if (r.flags.ims) {
 809         // handle If-Modified-Since requests from the client
 810         if (e->modifiedSince(&r)) {
 811             http->logType = LOG_TCP_IMS_HIT;
 812             sendMoreData(result);
 813             return;
 814         }
 815
 816         if (matchedIfNoneMatch) {
 817             // If-None-Match matched, reply with 304 Not Modified or
 818             // 412 Precondition Failed
 819             sendNotModifiedOrPreconditionFailedError();
 820             return;
 821         }
 822
 823         // otherwise reply with 304 Not Modified
 824         sendNotModified();
 825     }
 826 }
 827
 828 /// whether squid.conf send_hit prevents us from serving this hit
 829 bool
 830 clientReplyContext::blockedHit() const
 831 {
 832     if (!Config.accessList.sendHit)
 833         return false; // hits are not blocked by default
 834
 835     if (http->flags.internal)
 836         return false; // internal content "hits" cannot be blocked
 837
 838     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 839         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 840         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 841         HTTPMSGLOCK(chl->reply);
 842         return chl->fastCheck() != ACCESS_ALLOWED; // when in doubt, block
 843     }
 844
 845     // This does not happen, I hope, because we are called from CacheHit, which
 846     // is called via a storeClientCopy() callback, and store should initialize
 847     // the reply before calling that callback.
 848     debugs(88, 3, "Missing reply!");
 849     return false;
 850 }
 851
 852 void
 853 clientReplyContext::purgeRequestFindObjectToPurge()
 854 {
 855     /* Try to find a base entry */
 856     http->flags.purging = true;
 857     lookingforstore = 1;
 858
 859     // TODO: can we use purgeAllCached() here instead of doing the
 860     // getPublicByRequestMethod() dance?
 861     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 862 }
 863
 864 // Purges all entries with a given url
 865 // TODO: move to SideAgent parent, when we have one
 866 /*
 867  * We probably cannot purge Vary-affected responses because their MD5
 868  * keys depend on vary headers.
 869  */
 870 void
 871 purgeEntriesByUrl(HttpRequest * req, const char *url)
 872 {
 873 #if USE_HTCP
 874     bool get_or_head_sent = false;
 875 #endif
 876
 877     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 878         if (m.respMaybeCacheable()) {
 879             if (StoreEntry *entry = storeGetPublic(url, m)) {
 880                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 881 #if USE_HTCP
 882                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 883                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 884                     get_or_head_sent = true;
 885                 }
 886 #endif
 887                 entry->release();
 888             }
 889         }
 890     }
 891
 892 #if USE_HTCP
 893     if (!get_or_head_sent) {
 894         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 895     }
 896 #endif
 897 }
 898
 899 void
 900 clientReplyContext::purgeAllCached()
 901 {
 902     // XXX: performance regression, c_str() reallocates
 903     SBuf url(http->request->effectiveRequestUri());
 904     purgeEntriesByUrl(http->request, url.c_str());
 905 }
 906
 907 void
 908 clientReplyContext::created(StoreEntry *newEntry)
 909 {
 910     if (lookingforstore == 1)
 911         purgeFoundGet(newEntry);
 912     else if (lookingforstore == 2)
 913         purgeFoundHead(newEntry);
 914     else if (lookingforstore == 3)
 915         purgeDoPurgeGet(newEntry);
 916     else if (lookingforstore == 4)
 917         purgeDoPurgeHead(newEntry);
 918     else if (lookingforstore == 5)
 919         identifyFoundObject(newEntry);
 920 }
 921
 922 void
 923 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 924 {
 925     if (newEntry->isNull()) {
 926         lookingforstore = 2;
 927         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 928     } else
 929         purgeFoundObject (newEntry);
 930 }
 931
 932 void
 933 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 934 {
 935     if (newEntry->isNull())
 936         purgeDoMissPurge();
 937     else
 938         purgeFoundObject (newEntry);
 939 }
 940
 941 void
 942 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 943 {
 944     assert (entry && !entry->isNull());
 945
 946     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 947         http->logType = LOG_TCP_DENIED;
 948         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 949                                            http->getConn()->clientConnection->remote, http->request);
 950         startError(err);
 951         return; // XXX: leaking unused entry if some store does not keep it
 952     }
 953
 954     StoreIOBuffer localTempBuffer;
 955     /* Swap in the metadata */
 956     http->storeEntry(entry);
 957
 958     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 959     http->storeEntry()->createMemObject(storeId(), http->log_uri,
 960                                         http->request->method);
 961
 962     sc = storeClientListAdd(http->storeEntry(), this);
 963
 964     http->logType = LOG_TCP_HIT;
 965
 966     reqofs = 0;
 967
 968     localTempBuffer.offset = http->out.offset;
 969
 970     localTempBuffer.length = next()->readBuffer.length;
 971
 972     localTempBuffer.data = next()->readBuffer.data;
 973
 974     storeClientCopy(sc, http->storeEntry(),
 975                     localTempBuffer, CacheHit, this);
 976 }
 977
 978 void
 979 clientReplyContext::purgeRequest()
 980 {
 981     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 982            Config2.onoff.enable_purge);
 983
 984     if (!Config2.onoff.enable_purge) {
 985         http->logType = LOG_TCP_DENIED;
 986         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 987         startError(err);
 988         return;
 989     }
 990
 991     /* Release both IP cache */
 992     ipcacheInvalidate(http->request->url.host());
 993
 994     if (!http->flags.purging)
 995         purgeRequestFindObjectToPurge();
 996     else
 997         purgeDoMissPurge();
 998 }
 999
1000 void
1001 clientReplyContext::purgeDoMissPurge()
1002 {
1003     http->logType = LOG_TCP_MISS;
1004     lookingforstore = 3;
1005     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
1006 }
1007
1008 void
1009 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
1010 {
1011     assert (newEntry);
1012     /* Move to new() when that is created */
1013     purgeStatus = Http::scNotFound;
1014
1015     if (!newEntry->isNull()) {
1016         /* Release the cached URI */
1017         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
1018 #if USE_HTCP
1019         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1020 #endif
1021         newEntry->release();
1022         purgeStatus = Http::scOkay;
1023     }
1024
1025     lookingforstore = 4;
1026     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
1027 }
1028
1029 void
1030 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
1031 {
1032     if (newEntry && !newEntry->isNull()) {
1033         debugs(88, 4, "HEAD " << newEntry->url());
1034 #if USE_HTCP
1035         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1036 #endif
1037         newEntry->release();
1038         purgeStatus = Http::scOkay;
1039     }
1040
1041     /* And for Vary, release the base URI if none of the headers was included in the request */
1042     if (!http->request->vary_headers.isEmpty()
1043             && http->request->vary_headers.find('=') != SBuf::npos) {
1044         // XXX: performance regression, c_str() reallocates
1045         SBuf tmp(http->request->effectiveRequestUri());
1046         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1047
1048         if (entry) {
1049             debugs(88, 4, "Vary GET " << entry->url());
1050 #if USE_HTCP
1051             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1052 #endif
1053             entry->release();
1054             purgeStatus = Http::scOkay;
1055         }
1056
1057         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1058
1059         if (entry) {
1060             debugs(88, 4, "Vary HEAD " << entry->url());
1061 #if USE_HTCP
1062             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1063 #endif
1064             entry->release();
1065             purgeStatus = Http::scOkay;
1066         }
1067     }
1068
1069     /*
1070      * Make a new entry to hold the reply to be written
1071      * to the client.
1072      */
1073     /* FIXME: This doesn't need to go through the store. Simply
1074      * push down the client chain
1075      */
1076     createStoreEntry(http->request->method, RequestFlags());
1077
1078     triggerInitialStoreRead();
1079
1080     HttpReply *rep = new HttpReply;
1081     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1082     http->storeEntry()->replaceHttpReply(rep);
1083     http->storeEntry()->complete();
1084 }
1085
1086 void
1087 clientReplyContext::traceReply(clientStreamNode * node)
1088 {
1089     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1090     StoreIOBuffer localTempBuffer;
1091     createStoreEntry(http->request->method, RequestFlags());
1092     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1093     localTempBuffer.length = nextNode->readBuffer.length;
1094     localTempBuffer.data = nextNode->readBuffer.data;
1095     storeClientCopy(sc, http->storeEntry(),
1096                     localTempBuffer, SendMoreData, this);
1097     http->storeEntry()->releaseRequest();
1098     http->storeEntry()->buffer();
1099     HttpReply *rep = new HttpReply;
1100     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1101     http->storeEntry()->replaceHttpReply(rep);
1102     http->request->swapOut(http->storeEntry());
1103     http->storeEntry()->complete();
1104 }
1105
1106 #define SENDING_BODY 0
1107 #define SENDING_HDRSONLY 1
1108 int
1109 clientReplyContext::checkTransferDone()
1110 {
1111     StoreEntry *entry = http->storeEntry();
1112
1113     if (entry == NULL)
1114         return 0;
1115
1116     /*
1117      * For now, 'done_copying' is used for special cases like
1118      * Range and HEAD requests.
1119      */
1120     if (http->flags.done_copying)
1121         return 1;
1122
1123     if (http->request->flags.chunkedReply && !flags.complete) {
1124         // last-chunk was not sent
1125         return 0;
1126     }
1127
1128     /*
1129      * Handle STORE_OK objects.
1130      * objectLen(entry) will be set proprely.
1131      * RC: Does objectLen(entry) include the Headers?
1132      * RC: Yes.
1133      */
1134     if (entry->store_status == STORE_OK) {
1135         return storeOKTransferDone();
1136     } else {
1137         return storeNotOKTransferDone();
1138     }
1139 }
1140
1141 int
1142 clientReplyContext::storeOKTransferDone() const
1143 {
1144     assert(http->storeEntry()->objectLen() >= 0);
1145     assert(http->storeEntry()->objectLen() >= headers_sz);
1146     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1147         debugs(88,3,HERE << "storeOKTransferDone " <<
1148                " out.offset=" << http->out.offset <<
1149                " objectLen()=" << http->storeEntry()->objectLen() <<
1150                " headers_sz=" << headers_sz);
1151         return 1;
1152     }
1153
1154     return 0;
1155 }
1156
1157 int
1158 clientReplyContext::storeNotOKTransferDone() const
1159 {
1160     /*
1161      * Now, handle STORE_PENDING objects
1162      */
1163     MemObject *mem = http->storeEntry()->mem_obj;
1164     assert(mem != NULL);
1165     assert(http->request != NULL);
1166     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1167     HttpReply const *curReply = mem->getReply();
1168
1169     if (headers_sz == 0)
1170         /* haven't found end of headers yet */
1171         return 0;
1172
1173     /*
1174      * Figure out how much data we are supposed to send.
1175      * If we are sending a body and we don't have a content-length,
1176      * then we must wait for the object to become STORE_OK.
1177      */
1178     if (curReply->content_length < 0)
1179         return 0;
1180
1181     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1182
1183     if (http->out.size < expectedLength)
1184         return 0;
1185     else {
1186         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1187                " out.size=" << http->out.size <<
1188                " expectedLength=" << expectedLength);
1189         return 1;
1190     }
1191 }
1192
1193 /* A write has completed, what is the next status based on the
1194  * canonical request data?
1195  * 1 something is wrong
1196  * 0 nothing is wrong.
1197  *
1198  */
1199 int
1200 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1201 {
1202 #if SIZEOF_INT64_T == 4
1203     if (http->out.size > 0x7FFF0000) {
1204         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1205         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1206         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1207         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1208         return 1;
1209     }
1210
1211     if (http->out.offset > 0x7FFF0000) {
1212         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1213         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1214         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1215         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1216         return 1;
1217     }
1218
1219 #endif
1220     return 0;
1221 }
1222
1223 /* Preconditions:
1224  * *http is a valid structure.
1225  * fd is either -1, or an open fd.
1226  *
1227  * TODO: enumify this
1228  *
1229  * This function is used by any http request sink, to determine the status
1230  * of the object.
1231  */
1232 clientStream_status_t
1233 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1234 {
1235     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1236     assert (context);
1237     assert (context->http == http);
1238     return context->replyStatus();
1239 }
1240
1241 clientStream_status_t
1242 clientReplyContext::replyStatus()
1243 {
1244     int done;
1245     /* Here because lower nodes don't need it */
1246
1247     if (http->storeEntry() == NULL) {
1248         debugs(88, 5, "clientReplyStatus: no storeEntry");
1249         return STREAM_FAILED;   /* yuck, but what can we do? */
1250     }
1251
1252     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1253         /* TODO: Could upstream read errors (result.flags.error) be
1254          * lost, and result in undersize requests being considered
1255          * complete. Should we tcp reset such connections ?
1256          */
1257         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1258         return STREAM_FAILED;
1259     }
1260
1261     if ((done = checkTransferDone()) != 0 || flags.complete) {
1262         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1263         /* Ok we're finished, but how? */
1264
1265         const int64_t expectedBodySize =
1266             http->storeEntry()->getReply()->bodySize(http->request->method);
1267         if (!http->request->flags.proxyKeepalive && expectedBodySize < 0) {
1268             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
1269             return STREAM_FAILED;
1270         }
1271
1272         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1273             debugs(88, 5, "clientReplyStatus: truncated response body");
1274             return STREAM_UNPLANNED_COMPLETE;
1275         }
1276
1277         if (!done) {
1278             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1279             return STREAM_FAILED;
1280         }
1281
1282         if (expectedBodySize >= 0 && !http->gotEnough()) {
1283             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1284             return STREAM_UNPLANNED_COMPLETE;
1285         }
1286
1287         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1288                http->request->flags.proxyKeepalive);
1289         return STREAM_COMPLETE;
1290     }
1291
1292     // XXX: Should this be checked earlier? We could return above w/o checking.
1293     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1294         /* 4096 is a margin for the HTTP headers included in out.offset */
1295         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1296         return STREAM_FAILED;
1297     }
1298
1299     return STREAM_NONE;
1300 }
1301
1302 /* Responses with no body will not have a content-type header,
1303  * which breaks the rep_mime_type acl, which
1304  * coincidentally, is the most common acl for reply access lists.
1305  * A better long term fix for this is to allow acl matchs on the various
1306  * status codes, and then supply a default ruleset that puts these
1307  * codes before any user defines access entries. That way the user
1308  * can choose to block these responses where appropriate, but won't get
1309  * mysterious breakages.
1310  */
1311 bool
1312 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1313 {
1314     bool result;
1315
1316     switch (sline) {
1317
1318     case Http::scContinue:
1319
1320     case Http::scSwitchingProtocols:
1321
1322     case Http::scProcessing:
1323
1324     case Http::scNoContent:
1325
1326     case Http::scNotModified:
1327         result = true;
1328         break;
1329
1330     default:
1331         result = false;
1332     }
1333
1334     return result;
1335 }
1336
1337 /**
1338  * Generate the reply headers sent to client.
1339  *
1340  * Filters out unwanted entries and hop-by-hop from original reply header
1341  * then adds extra entries if we have more info than origin server
1342  * then adds Squid specific entries
1343  */
1344 void
1345 clientReplyContext::buildReplyHeader()
1346 {
1347     HttpHeader *hdr = &reply->header;
1348     const bool is_hit = http->logType.isTcpHit();
1349     HttpRequest *request = http->request;
1350 #if DONT_FILTER_THESE
1351     /* but you might want to if you run Squid as an HTTP accelerator */
1352     /* hdr->delById(HDR_ACCEPT_RANGES); */
1353     hdr->delById(HDR_ETAG);
1354 #endif
1355
1356     if (is_hit)
1357         hdr->delById(Http::HdrType::SET_COOKIE);
1358     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1359
1360     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1361     // remove the Proxy-Authenticate header
1362     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0))
1363         reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1364
1365     reply->header.removeHopByHopEntries();
1366
1367     //    if (request->range)
1368     //      clientBuildRangeHeader(http, reply);
1369
1370     /*
1371      * Add a estimated Age header on cache hits.
1372      */
1373     if (is_hit) {
1374         /*
1375          * Remove any existing Age header sent by upstream caches
1376          * (note that the existing header is passed along unmodified
1377          * on cache misses)
1378          */
1379         hdr->delById(Http::HdrType::AGE);
1380         /*
1381          * This adds the calculated object age. Note that the details of the
1382          * age calculation is performed by adjusting the timestamp in
1383          * StoreEntry::timestampsSet(), not here.
1384          */
1385         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1386             hdr->delById(Http::HdrType::DATE);
1387             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1388         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1389             // Swap the Date: header to current time if we are simulating an origin
1390             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1391             if (h)
1392                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1393             hdr->delById(Http::HdrType::DATE);
1394             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1395             h = hdr->findEntry(Http::HdrType::EXPIRES);
1396             if (h && http->storeEntry()->expires >= 0) {
1397                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1398                 hdr->delById(Http::HdrType::EXPIRES);
1399                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1400             }
1401             if (http->storeEntry()->timestamp <= squid_curtime) {
1402                 // put X-Cache-Age: instead of Age:
1403                 char age[64];
1404                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1405                 hdr->putExt("X-Cache-Age", age);
1406             }
1407         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1408             hdr->putInt(Http::HdrType::AGE,
1409                         squid_curtime - http->storeEntry()->timestamp);
1410             /* Signal old objects.  NB: rfc 2616 is not clear,
1411              * by implication, on whether we should do this to all
1412              * responses, or only cache hits.
1413              * 14.46 states it ONLY applys for heuristically caclulated
1414              * freshness values, 13.2.4 doesn't specify the same limitation.
1415              * We interpret RFC 2616 under the combination.
1416              */
1417             /* TODO: if maxage or s-maxage is present, don't do this */
1418
1419             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1420                 char tbuf[512];
1421                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1422                           "113", ThisCache,
1423                           "This cache hit is still fresh and more than 1 day old");
1424                 hdr->putStr(Http::HdrType::WARNING, tbuf);
1425             }
1426         }
1427     }
1428
1429     /* RFC 2616: Section 14.18
1430      *
1431      * Add a Date: header if missing.
1432      * We have access to a clock therefore are required to amend any shortcoming in servers.
1433      *
1434      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1435      */
1436     if ( !hdr->has(Http::HdrType::DATE) ) {
1437         if (!http->storeEntry())
1438             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1439         else if (http->storeEntry()->timestamp > 0)
1440             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1441         else {
1442             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1443             /* dump something useful about the problem */
1444             http->storeEntry()->dump(DBG_IMPORTANT);
1445         }
1446     }
1447
1448     // add Warnings required by RFC 2616 if serving a stale hit
1449     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1450         hdr->putWarning(110, "Response is stale");
1451         if (http->request->flags.needValidation)
1452             hdr->putWarning(111, "Revalidation failed");
1453     }
1454
1455     /* Filter unproxyable authentication types */
1456     if (http->logType.oldType != LOG_TCP_DENIED &&
1457             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1458         HttpHeaderPos pos = HttpHeaderInitPos;
1459         HttpHeaderEntry *e;
1460
1461         int connection_auth_blocked = 0;
1462         while ((e = hdr->getEntry(&pos))) {
1463             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1464                 const char *value = e->value.rawBuf();
1465
1466                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1467                         (value[4] == '\0' || value[4] == ' '))
1468                         ||
1469                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1470                          (value[9] == '\0' || value[9] == ' '))
1471                         ||
1472                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1473                          (value[8] == '\0' || value[8] == ' '))) {
1474                     if (request->flags.connectionAuthDisabled) {
1475                         hdr->delAt(pos, connection_auth_blocked);
1476                         continue;
1477                     }
1478                     request->flags.mustKeepalive = true;
1479                     if (!request->flags.accelerated && !request->flags.intercepted) {
1480                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1481                         /*
1482                           We send "Connection: Proxy-Support" header to mark
1483                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1484                           understand the semantics of this header. The RFC should have included
1485                           this recommendation.
1486                         */
1487                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1488                     }
1489                     break;
1490                 }
1491             }
1492         }
1493
1494         if (connection_auth_blocked)
1495             hdr->refreshMask();
1496     }
1497
1498 #if USE_AUTH
1499     /* Handle authentication headers */
1500     if (http->logType.oldType == LOG_TCP_DENIED &&
1501             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1502               reply->sline.status() == Http::scUnauthorized)
1503        ) {
1504         /* Add authentication header */
1505         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1506          * depends on authenticate behaviour: all schemes to date send no extra
1507          * data on 407/401 responses, and do not check the accel state on 401/407
1508          * responses
1509          */
1510         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1511     } else if (request->auth_user_request != NULL)
1512         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1513 #endif
1514
1515     /* Append X-Cache */
1516     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1517                       is_hit ? "HIT" : "MISS", getMyHostname());
1518
1519 #if USE_CACHE_DIGESTS
1520     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1521     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1522                       lookup_type ? lookup_type : "NONE",
1523                       getMyHostname(), getMyPort());
1524
1525 #endif
1526
1527     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1528                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1529                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1530
1531     /* Check whether we should send keep-alive */
1532     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1533         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1534         request->flags.proxyKeepalive = false;
1535     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1536         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1537         request->flags.proxyKeepalive = false;
1538     } else if (request->flags.proxyKeepalive && shutting_down) {
1539         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1540         request->flags.proxyKeepalive = false;
1541     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1542         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1543         request->flags.proxyKeepalive = false;
1544     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1545         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1546         request->flags.proxyKeepalive = false;
1547     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1548         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1549         request->flags.proxyKeepalive = false;
1550     } else if (request->flags.sslBumped && !reply->persistent()) {
1551         // We do not really have to close, but we pretend we are a tunnel.
1552         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1553         request->flags.proxyKeepalive = false;
1554     } else if (request->pinnedConnection() && !reply->persistent()) {
1555         // The peer wants to close the pinned connection
1556         debugs(88, 3, "pinned reply forces close");
1557         request->flags.proxyKeepalive = false;
1558     } else if (http->getConn()) {
1559         ConnStateData * conn = http->getConn();
1560         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1561             // The listening port closed because of a reconfigure
1562             debugs(88, 3, "listening port closed");
1563             request->flags.proxyKeepalive = false;
1564         }
1565     }
1566
1567     // Decide if we send chunked reply
1568     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1569         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1570         request->flags.chunkedReply = true;
1571         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1572     }
1573
1574     /* Append VIA */
1575     if (Config.onoff.via) {
1576         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1577         String strVia;
1578         hdr->getList(Http::HdrType::VIA, &strVia);
1579         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1580                  reply->sline.version.major,
1581                  reply->sline.version.minor,
1582                  ThisCache);
1583         strListAdd(&strVia, bbuf, ',');
1584         hdr->delById(Http::HdrType::VIA);
1585         hdr->putStr(Http::HdrType::VIA, strVia.termedBuf());
1586     }
1587     /* Signal keep-alive or close explicitly */
1588     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1589
1590 #if ADD_X_REQUEST_URI
1591     /*
1592      * Knowing the URI of the request is useful when debugging persistent
1593      * connections in a client; we cannot guarantee the order of http headers,
1594      * but X-Request-URI is likely to be the very last header to ease use from a
1595      * debugger [hdr->entries.count-1].
1596      */
1597     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1598                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1599
1600 #endif
1601
1602     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1603     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1604         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1605             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1606         }
1607         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1608     }
1609
1610     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1611 }
1612
1613 void
1614 clientReplyContext::cloneReply()
1615 {
1616     assert(reply == NULL);
1617
1618     reply = http->storeEntry()->getReply()->clone();
1619     HTTPMSGLOCK(reply);
1620
1621     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1622         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1623         reply->sline.version = Http::ProtocolVersion();
1624     }
1625
1626     /* do header conversions */
1627     buildReplyHeader();
1628 }
1629
1630 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1631 /// We cannot just ignore the entry because it may be locking or otherwise
1632 /// holding an associated cache resource of some sort.
1633 void
1634 clientReplyContext::forgetHit()
1635 {
1636     StoreEntry *e = http->storeEntry();
1637     assert(e); // or we are not dealing with a hit
1638     // We probably have not locked the entry earlier, unfortunately. We lock it
1639     // now so that we can unlock two lines later (and trigger cleanup).
1640     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1641     // used so inconsistently that simply adding locking there leads to bugs.
1642     e->lock("clientReplyContext::forgetHit");
1643     http->storeEntry(NULL);
1644     e->unlock("clientReplyContext::forgetHit"); // may delete e
1645 }
1646
1647 void
1648 clientReplyContext::identifyStoreObject()
1649 {
1650     HttpRequest *r = http->request;
1651
1652     if (r->flags.cachable || r->flags.internal) {
1653         lookingforstore = 5;
1654         StoreEntry::getPublicByRequest (this, r);
1655     } else {
1656         identifyFoundObject (NullStoreEntry::getInstance());
1657     }
1658 }
1659
1660 /**
1661  * Check state of the current StoreEntry object.
1662  * to see if we can determine the final status of the request.
1663  */
1664 void
1665 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1666 {
1667     StoreEntry *e = newEntry;
1668     HttpRequest *r = http->request;
1669
1670     /** \li If the entry received isNull() then we ignore it. */
1671     if (e->isNull()) {
1672         http->storeEntry(NULL);
1673     } else {
1674         http->storeEntry(e);
1675     }
1676
1677     e = http->storeEntry();
1678
1679     /* Release IP-cache entries on reload */
1680     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1681       * 'invalidate' the cached IP entries for this request ???
1682       */
1683     if (r->flags.noCache || r->flags.noCacheHack())
1684         ipcacheInvalidateNegative(r->url.host());
1685
1686 #if USE_CACHE_DIGESTS
1687     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1688 #endif
1689
1690     if (NULL == http->storeEntry()) {
1691         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1692         debugs(85, 3, "StoreEntry is NULL -  MISS");
1693         http->logType = LOG_TCP_MISS;
1694         doGetMoreData();
1695         return;
1696     }
1697
1698     if (Config.onoff.offline) {
1699         /** \li If we are running in offline mode set to HIT */
1700         debugs(85, 3, "offline HIT " << *e);
1701         http->logType = LOG_TCP_HIT;
1702         doGetMoreData();
1703         return;
1704     }
1705
1706     if (http->redirect.status) {
1707         /** \li If redirection status is True force this to be a MISS */
1708         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1709         forgetHit();
1710         http->logType = LOG_TCP_REDIRECT;
1711         doGetMoreData();
1712         return;
1713     }
1714
1715     if (!e->validToSend()) {
1716         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1717         forgetHit();
1718         http->logType = LOG_TCP_MISS;
1719         doGetMoreData();
1720         return;
1721     }
1722
1723     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1724         /* \li Special entries are always hits, no matter what the client says */
1725         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1726         http->logType = LOG_TCP_HIT;
1727         doGetMoreData();
1728         return;
1729     }
1730
1731     if (r->flags.noCache) {
1732         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1733         forgetHit();
1734         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1735         doGetMoreData();
1736         return;
1737     }
1738
1739     debugs(85, 3, "default HIT " << *e);
1740     http->logType = LOG_TCP_HIT;
1741     doGetMoreData();
1742 }
1743
1744 /**
1745  * Request more data from the store for the client Stream
1746  * This is *the* entry point to this module.
1747  *
1748  * Preconditions:
1749  *  - This is the head of the list.
1750  *  - There is at least one more node.
1751  *  - Data context is not null
1752  */
1753 void
1754 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1755 {
1756     /* Test preconditions */
1757     assert(aNode != NULL);
1758     assert(cbdataReferenceValid(aNode));
1759     assert(aNode->node.prev == NULL);
1760     assert(aNode->node.next != NULL);
1761     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1762     assert (context);
1763     assert(context->http == http);
1764
1765     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1766
1767     if (!context->ourNode)
1768         context->ourNode = aNode;
1769
1770     /* no cbdatareference, this is only used once, and safely */
1771     if (context->flags.storelogiccomplete) {
1772         StoreIOBuffer tempBuffer;
1773         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1774         tempBuffer.length = next->readBuffer.length;
1775         tempBuffer.data = next->readBuffer.data;
1776
1777         storeClientCopy(context->sc, http->storeEntry(),
1778                         tempBuffer, clientReplyContext::SendMoreData, context);
1779         return;
1780     }
1781
1782     if (context->http->request->method == Http::METHOD_PURGE) {
1783         context->purgeRequest();
1784         return;
1785     }
1786
1787     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1788
1789     if (context->http->request->method == Http::METHOD_TRACE) {
1790         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1791             context->traceReply(aNode);
1792             return;
1793         }
1794
1795         /* continue forwarding, not finished yet. */
1796         http->logType = LOG_TCP_MISS;
1797
1798         context->doGetMoreData();
1799     } else
1800         context->identifyStoreObject();
1801 }
1802
1803 void
1804 clientReplyContext::doGetMoreData()
1805 {
1806     /* We still have to do store logic processing - vary, cache hit etc */
1807     if (http->storeEntry() != NULL) {
1808         /* someone found the object in the cache for us */
1809         StoreIOBuffer localTempBuffer;
1810
1811         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1812
1813         MemObject *mem_obj = http->storeEntry()->makeMemObject();
1814         if (!mem_obj->hasUris()) {
1815             /*
1816              * This if-block exists because we don't want to clobber
1817              * a preexiting mem_obj->method value if the mem_obj
1818              * already exists.  For example, when a HEAD request
1819              * is a cache hit for a GET response, we want to keep
1820              * the method as GET.
1821              */
1822             mem_obj->setUris(storeId(), http->log_uri, http->request->method);
1823             /**
1824              * Here we can see if the object was
1825              * created using URL or alternative StoreID from helper.
1826              */
1827             debugs(88, 3, "storeId: " << http->storeEntry()->mem_obj->storeId());
1828         }
1829
1830         sc = storeClientListAdd(http->storeEntry(), this);
1831 #if USE_DELAY_POOLS
1832         sc->setDelayId(DelayId::DelayClient(http));
1833 #endif
1834
1835         assert(http->logType.oldType == LOG_TCP_HIT);
1836         reqofs = 0;
1837         /* guarantee nothing has been sent yet! */
1838         assert(http->out.size == 0);
1839         assert(http->out.offset == 0);
1840
1841         if (Ip::Qos::TheConfig.isHitTosActive()) {
1842             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1843         }
1844
1845         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1846             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1847         }
1848
1849         localTempBuffer.offset = reqofs;
1850         localTempBuffer.length = getNextNode()->readBuffer.length;
1851         localTempBuffer.data = getNextNode()->readBuffer.data;
1852         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1853     } else {
1854         /* MISS CASE, http->logType is already set! */
1855         processMiss();
1856     }
1857 }
1858
1859 /** The next node has removed itself from the stream. */
1860 void
1861 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1862 {
1863     /** detach from the stream */
1864     clientStreamDetach(node, http);
1865 }
1866
1867 /**
1868  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1869  * such, writes processed message to the message recipient
1870  */
1871 void
1872 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1873 {
1874     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1875     context->sendMoreData (result);
1876 }
1877
1878 void
1879 clientReplyContext::makeThisHead()
1880 {
1881     /* At least, I think thats what this does */
1882     dlinkDelete(&http->active, &ClientActiveRequests);
1883     dlinkAdd(http, &http->active, &ClientActiveRequests);
1884 }
1885
1886 bool
1887 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1888 {
1889     return /* aborted request */
1890         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1891         /* Upstream read error */ (result.flags.error) ||
1892         /* Upstream EOF */ (sizeToProcess == 0);
1893 }
1894
1895 void
1896 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1897 {
1898     /** call clientWriteComplete so the client socket gets closed
1899      *
1900      * We call into the stream, because we don't know that there is a
1901      * client socket!
1902      */
1903     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1904     StoreIOBuffer localTempBuffer;
1905     flags.complete = 1;
1906     http->request->flags.streamError = true;
1907     localTempBuffer.flags.error = result.flags.error;
1908     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1909                          localTempBuffer);
1910 }
1911
1912 void
1913 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1914 {
1915     StoreIOBuffer localTempBuffer;
1916
1917     if (result.length == 0) {
1918         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1919         flags.complete = 1;
1920     }
1921
1922     assert(result.offset - headers_sz == next()->readBuffer.offset);
1923     localTempBuffer.offset = result.offset - headers_sz;
1924     localTempBuffer.length = result.length;
1925
1926     if (localTempBuffer.length)
1927         localTempBuffer.data = source;
1928
1929     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1930                          localTempBuffer);
1931 }
1932
1933 clientStreamNode *
1934 clientReplyContext::next() const
1935 {
1936     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1937     return getNextNode();
1938 }
1939
1940 void
1941 clientReplyContext::sendBodyTooLargeError()
1942 {
1943     Ip::Address tmp_noaddr;
1944     tmp_noaddr.setNoAddr(); // TODO: make a global const
1945     http->logType = LOG_TCP_DENIED_REPLY;
1946     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1947                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1948                                        http->request);
1949     removeClientStoreReference(&(sc), http);
1950     HTTPMSGUNLOCK(reply);
1951     startError(err);
1952
1953 }
1954
1955 /// send 412 (Precondition Failed) to client
1956 void
1957 clientReplyContext::sendPreconditionFailedError()
1958 {
1959     http->logType = LOG_TCP_HIT;
1960     ErrorState *const err =
1961         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1962                          NULL, http->getConn()->clientConnection->remote, http->request);
1963     removeClientStoreReference(&sc, http);
1964     HTTPMSGUNLOCK(reply);
1965     startError(err);
1966 }
1967
1968 /// send 304 (Not Modified) to client
1969 void
1970 clientReplyContext::sendNotModified()
1971 {
1972     StoreEntry *e = http->storeEntry();
1973     const time_t timestamp = e->timestamp;
1974     HttpReply *const temprep = e->getReply()->make304();
1975     http->logType = LOG_TCP_IMS_HIT;
1976     removeClientStoreReference(&sc, http);
1977     createStoreEntry(http->request->method, RequestFlags());
1978     e = http->storeEntry();
1979     // Copy timestamp from the original entry so the 304
1980     // reply has a meaningful Age: header.
1981     e->timestampsSet();
1982     e->timestamp = timestamp;
1983     e->replaceHttpReply(temprep);
1984     e->complete();
1985     /*
1986      * TODO: why put this in the store and then serialise it and
1987      * then parse it again. Simply mark the request complete in
1988      * our context and write the reply struct to the client side.
1989      */
1990     triggerInitialStoreRead();
1991 }
1992
1993 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1994 /// depending on request method
1995 void
1996 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1997 {
1998     if (http->request->method == Http::METHOD_GET ||
1999             http->request->method == Http::METHOD_HEAD)
2000         sendNotModified();
2001     else
2002         sendPreconditionFailedError();
2003 }
2004
2005 void
2006 clientReplyContext::processReplyAccess ()
2007 {
2008     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
2009     assert(reply);
2010
2011     /** Don't block our own responses or HTTP status messages */
2012     if (http->logType.oldType == LOG_TCP_DENIED ||
2013             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
2014             alwaysAllowResponse(reply->sline.status())) {
2015         headers_sz = reply->hdr_sz;
2016         processReplyAccessResult(ACCESS_ALLOWED);
2017         return;
2018     }
2019
2020     /** Check for reply to big error */
2021     if (reply->expectedBodyTooLarge(*http->request)) {
2022         sendBodyTooLargeError();
2023         return;
2024     }
2025
2026     headers_sz = reply->hdr_sz;
2027
2028     /** check for absent access controls (permit by default) */
2029     if (!Config.accessList.reply) {
2030         processReplyAccessResult(ACCESS_ALLOWED);
2031         return;
2032     }
2033
2034     /** Process http_reply_access lists */
2035     ACLFilledChecklist *replyChecklist =
2036         clientAclChecklistCreate(Config.accessList.reply, http);
2037     replyChecklist->reply = reply;
2038     HTTPMSGLOCK(replyChecklist->reply);
2039     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2040 }
2041
2042 void
2043 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2044 {
2045     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2046     me->processReplyAccessResult(rv);
2047 }
2048
2049 void
2050 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2051 {
2052     debugs(88, 2, "The reply for " << http->request->method
2053            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2054            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2055
2056     if (accessAllowed != ACCESS_ALLOWED) {
2057         ErrorState *err;
2058         err_type page_id;
2059         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2060
2061         http->logType = LOG_TCP_DENIED_REPLY;
2062
2063         if (page_id == ERR_NONE)
2064             page_id = ERR_ACCESS_DENIED;
2065
2066         Ip::Address tmp_noaddr;
2067         tmp_noaddr.setNoAddr();
2068         err = clientBuildError(page_id, Http::scForbidden, NULL,
2069                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2070                                http->request);
2071
2072         removeClientStoreReference(&sc, http);
2073
2074         HTTPMSGUNLOCK(reply);
2075
2076         startError(err);
2077
2078         return;
2079     }
2080
2081     /* Ok, the reply is allowed, */
2082     http->loggingEntry(http->storeEntry());
2083
2084     ssize_t body_size = reqofs - reply->hdr_sz;
2085     if (body_size < 0) {
2086         reqofs = reply->hdr_sz;
2087         body_size = 0;
2088     }
2089
2090     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2091            (int) body_size << " bytes after " << reply->hdr_sz <<
2092            " bytes of headers");
2093
2094 #if USE_SQUID_ESI
2095
2096     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2097             !alwaysAllowResponse(reply->sline.status()) &&
2098             esiEnableProcessing(reply)) {
2099         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2100         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2101                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2102     }
2103
2104 #endif
2105
2106     if (http->request->method == Http::METHOD_HEAD) {
2107         /* do not forward body for HEAD replies */
2108         body_size = 0;
2109         http->flags.done_copying = true;
2110         flags.complete = 1;
2111     }
2112
2113     assert (!flags.headersSent);
2114     flags.headersSent = true;
2115
2116     StoreIOBuffer localTempBuffer;
2117     char *buf = next()->readBuffer.data;
2118     char *body_buf = buf + reply->hdr_sz;
2119
2120     //Server side may disable ranges under some circumstances.
2121
2122     if ((!http->request->range))
2123         next()->readBuffer.offset = 0;
2124
2125     body_buf -= next()->readBuffer.offset;
2126
2127     if (next()->readBuffer.offset != 0) {
2128         if (next()->readBuffer.offset > body_size) {
2129             /* Can't use any of the body we received. send nothing */
2130             localTempBuffer.length = 0;
2131             localTempBuffer.data = NULL;
2132         } else {
2133             localTempBuffer.length = body_size - next()->readBuffer.offset;
2134             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2135         }
2136     } else {
2137         localTempBuffer.length = body_size;
2138         localTempBuffer.data = body_buf;
2139     }
2140
2141     /* TODO??: move the data in the buffer back by the request header size */
2142     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2143                          http, reply, localTempBuffer);
2144
2145     return;
2146 }
2147
2148 void
2149 clientReplyContext::sendMoreData (StoreIOBuffer result)
2150 {
2151     if (deleting)
2152         return;
2153
2154     StoreEntry *entry = http->storeEntry();
2155
2156     if (ConnStateData * conn = http->getConn()) {
2157         if (!conn->isOpen()) {
2158             debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2159             return;
2160         }
2161         if (conn->pinning.zeroReply) {
2162             debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2163             return;
2164         }
2165
2166         if (reqofs==0 && !http->logType.isTcpHit()) {
2167             if (Ip::Qos::TheConfig.isHitTosActive()) {
2168                 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2169             }
2170             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2171                 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2172             }
2173         }
2174
2175         debugs(88, 5, conn->clientConnection <<
2176                " '" << entry->url() << "'" <<
2177                " out.offset=" << http->out.offset);
2178     }
2179
2180     char *buf = next()->readBuffer.data;
2181
2182     if (buf != result.data) {
2183         /* we've got to copy some data */
2184         assert(result.length <= next()->readBuffer.length);
2185         memcpy(buf, result.data, result.length);
2186     }
2187
2188     /* We've got the final data to start pushing... */
2189     flags.storelogiccomplete = 1;
2190
2191     reqofs += result.length;
2192
2193     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2194
2195     assert(http->request != NULL);
2196
2197     /* ESI TODO: remove this assert once everything is stable */
2198     assert(http->client_stream.head->data
2199            && cbdataReferenceValid(http->client_stream.head->data));
2200
2201     makeThisHead();
2202
2203     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2204            reqofs << " bytes (" << result.length <<
2205            " new bytes)");
2206
2207     /* update size of the request */
2208     reqsize = reqofs;
2209
2210     if (errorInStream(result, reqofs)) {
2211         sendStreamError(result);
2212         return;
2213     }
2214
2215     if (flags.headersSent) {
2216         pushStreamData (result, buf);
2217         return;
2218     }
2219
2220     cloneReply();
2221
2222 #if USE_DELAY_POOLS
2223     if (sc)
2224         sc->setDelayId(DelayId::DelayClient(http,reply));
2225 #endif
2226
2227     /* handle headers */
2228
2229     if (Config.onoff.log_mime_hdrs) {
2230         size_t k;
2231
2232         if ((k = headersEnd(buf, reqofs))) {
2233             safe_free(http->al->headers.reply);
2234             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2235             xstrncpy(http->al->headers.reply, buf, k);
2236         }
2237     }
2238
2239     holdingBuffer = result;
2240     processReplyAccess();
2241     return;
2242 }
2243
2244 /* Using this breaks the client layering just a little!
2245  */
2246 void
2247 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2248 {
2249     assert(http != NULL);
2250     /*
2251      * For erroneous requests, we might not have a h->request,
2252      * so make a fake one.
2253      */
2254
2255     if (http->request == NULL) {
2256         http->request = new HttpRequest(m, AnyP::PROTO_NONE, null_string);
2257         HTTPMSGLOCK(http->request);
2258     }
2259
2260     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2261
2262     // Make entry collapsable ASAP, to increase collapsing chances for others,
2263     // TODO: every must-revalidate and similar request MUST reach the origin,
2264     // but do we have to prohibit others from collapsing on that request?
2265     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2266             !reqFlags.needValidation &&
2267             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2268         // make the entry available for future requests now
2269         Store::Root().allowCollapsing(e, reqFlags, m);
2270     }
2271
2272     sc = storeClientListAdd(e, this);
2273
2274 #if USE_DELAY_POOLS
2275     sc->setDelayId(DelayId::DelayClient(http));
2276 #endif
2277
2278     reqofs = 0;
2279
2280     reqsize = 0;
2281
2282     /* I don't think this is actually needed! -- adrian */
2283     /* http->reqbuf = http->norm_reqbuf; */
2284     //    assert(http->reqbuf == http->norm_reqbuf);
2285     /* The next line is illegal because we don't know if the client stream
2286      * buffers have been set up
2287      */
2288     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2289     //        SendMoreData, this);
2290     /* So, we mark the store logic as complete */
2291     flags.storelogiccomplete = 1;
2292
2293     /* and get the caller to request a read, from whereever they are */
2294     /* NOTE: after ANY data flows down the pipe, even one step,
2295      * this function CAN NOT be used to manage errors
2296      */
2297     http->storeEntry(e);
2298 }
2299
2300 ErrorState *
2301 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2302                  Ip::Address &src_addr, HttpRequest * request)
2303 {
2304     ErrorState *err = new ErrorState(page_id, status, request);
2305     err->src_addr = src_addr;
2306
2307     if (url)
2308         err->url = xstrdup(url);
2309
2310     return err;
2311 }
2312