src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "http/Stream.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47 #if USE_SQUID_ESI
  48 #include "esi/Esi.h"
  49 #endif
  50
  51 #include <memory>
  52
  53 CBDATA_CLASS_INIT(clientReplyContext);
  54
  55 /* Local functions */
  56 extern "C" CSS clientReplyStatus;
  57 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  58
  59 /* privates */
  60
  61 clientReplyContext::~clientReplyContext()
  62 {
  63     deleting = true;
  64     /* This may trigger a callback back into SendMoreData as the cbdata
  65      * is still valid
  66      */
  67     removeClientStoreReference(&sc, http);
  68     /* old_entry might still be set if we didn't yet get the reply
  69      * code in HandleIMSReply() */
  70     removeStoreReference(&old_sc, &old_entry);
  71     safe_free(tempBuffer.data);
  72     cbdataReferenceDone(http);
  73     HTTPMSGUNLOCK(reply);
  74 }
  75
  76 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  77     purgeStatus(Http::scNone),
  78     lookingforstore(0),
  79     http(cbdataReference(clientContext)),
  80     headers_sz(0),
  81     sc(NULL),
  82     old_reqsize(0),
  83     reqsize(0),
  84     reqofs(0),
  85 #if USE_CACHE_DIGESTS
  86     lookup_type(NULL),
  87 #endif
  88     ourNode(NULL),
  89     reply(NULL),
  90     old_entry(NULL),
  91     old_sc(NULL),
  92     deleting(false)
  93 {
  94     *tempbuf = 0;
  95 }
  96
  97 /** Create an error in the store awaiting the client side to read it.
  98  *
  99  * This may be better placed in the clientStream logic, but it has not been
 100  * relocated there yet
 101  */
 102 void
 103 clientReplyContext::setReplyToError(
 104     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 105     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 106 #if USE_AUTH
 107     Auth::UserRequest::Pointer auth_user_request
 108 #else
 109     void*
 110 #endif
 111 )
 112 {
 113     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 114
 115     if (unparsedrequest)
 116         errstate->request_hdrs = xstrdup(unparsedrequest);
 117
 118 #if USE_AUTH
 119     errstate->auth_user_request = auth_user_request;
 120 #endif
 121     setReplyToError(method, errstate);
 122 }
 123
 124 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 125 {
 126     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 127         /* prevent confusion over whether we default to persistent or not */
 128         http->request->flags.proxyKeepalive = false;
 129
 130     http->al->http.code = errstate->httpStatus;
 131
 132     if (http->request)
 133         http->request->ignoreRange("responding with a Squid-generated error");
 134
 135     createStoreEntry(method, RequestFlags());
 136     assert(errstate->callback_data == NULL);
 137     errorAppendEntry(http->storeEntry(), errstate);
 138     /* Now the caller reads to get this */
 139 }
 140
 141 void
 142 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 143 {
 144     Must(futureReply);
 145     http->al->http.code = futureReply->sline.status();
 146
 147     HttpRequestMethod method;
 148     if (http->request) { // nil on responses to unparsable requests
 149         http->request->ignoreRange("responding with a Squid-generated reply");
 150         method = http->request->method;
 151     }
 152
 153     createStoreEntry(method, RequestFlags());
 154
 155     http->storeEntry()->storeErrorResponse(futureReply);
 156     /* Now the caller reads to get futureReply */
 157 }
 158
 159 // Assumes that the entry contains an error response without Content-Range.
 160 // To use with regular entries, make HTTP Range header removal conditional.
 161 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 162 {
 163     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 164     sc = storeClientListAdd(entry, this);
 165 #if USE_DELAY_POOLS
 166     sc->setDelayId(DelayId::DelayClient(http));
 167 #endif
 168     reqofs = 0;
 169     reqsize = 0;
 170     if (http->request)
 171         http->request->ignoreRange(reason);
 172     flags.storelogiccomplete = 1;
 173     http->storeEntry(entry);
 174 }
 175
 176 void
 177 clientReplyContext::removeStoreReference(store_client ** scp,
 178         StoreEntry ** ep)
 179 {
 180     StoreEntry *e;
 181     store_client *sc_tmp = *scp;
 182
 183     if ((e = *ep) != NULL) {
 184         *ep = NULL;
 185         storeUnregister(sc_tmp, e, this);
 186         *scp = NULL;
 187         e->unlock("clientReplyContext::removeStoreReference");
 188     }
 189 }
 190
 191 void
 192 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 193 {
 194     StoreEntry *reference = aHttpRequest->storeEntry();
 195     removeStoreReference(scp, &reference);
 196     aHttpRequest->storeEntry(reference);
 197 }
 198
 199 void
 200 clientReplyContext::saveState()
 201 {
 202     assert(old_sc == NULL);
 203     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 204     old_entry = http->storeEntry();
 205     old_sc = sc;
 206     old_reqsize = reqsize;
 207     tempBuffer.offset = reqofs;
 208     /* Prevent accessing the now saved entries */
 209     http->storeEntry(NULL);
 210     sc = NULL;
 211     reqsize = 0;
 212     reqofs = 0;
 213 }
 214
 215 void
 216 clientReplyContext::restoreState()
 217 {
 218     assert(old_sc != NULL);
 219     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 220     removeClientStoreReference(&sc, http);
 221     http->storeEntry(old_entry);
 222     sc = old_sc;
 223     reqsize = old_reqsize;
 224     reqofs = tempBuffer.offset;
 225     /* Prevent accessed the old saved entries */
 226     old_entry = NULL;
 227     old_sc = NULL;
 228     old_reqsize = 0;
 229     tempBuffer.offset = 0;
 230 }
 231
 232 void
 233 clientReplyContext::startError(ErrorState * err)
 234 {
 235     createStoreEntry(http->request->method, RequestFlags());
 236     triggerInitialStoreRead();
 237     errorAppendEntry(http->storeEntry(), err);
 238 }
 239
 240 clientStreamNode *
 241 clientReplyContext::getNextNode() const
 242 {
 243     return (clientStreamNode *)ourNode->node.next->data;
 244 }
 245
 246 /* This function is wrong - the client parameters don't include the
 247  * header offset
 248  */
 249 void
 250 clientReplyContext::triggerInitialStoreRead()
 251 {
 252     /* when confident, 0 becomes reqofs, and then this factors into
 253      * startSendProcess
 254      */
 255     assert(reqofs == 0);
 256     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 257     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 258 }
 259
 260 /* there is an expired entry in the store.
 261  * setup a temporary buffer area and perform an IMS to the origin
 262  */
 263 void
 264 clientReplyContext::processExpired()
 265 {
 266     const char *url = storeId();
 267     StoreEntry *entry = NULL;
 268     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 269     assert(http->storeEntry()->lastmod >= 0);
 270     /*
 271      * check if we are allowed to contact other servers
 272      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 273      *      a stale entry *if* it matches client requirements
 274      */
 275
 276     if (http->onlyIfCached()) {
 277         processOnlyIfCachedMiss();
 278         return;
 279     }
 280
 281     http->request->flags.refresh = true;
 282 #if STORE_CLIENT_LIST_DEBUG
 283     /* Prevent a race with the store client memory free routines
 284      */
 285     assert(storeClientIsThisAClient(sc, this));
 286 #endif
 287     /* Prepare to make a new temporary request */
 288     saveState();
 289     entry = storeCreateEntry(url,
 290                              http->log_uri, http->request->flags, http->request->method);
 291     /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 292     sc = storeClientListAdd(entry, this);
 293 #if USE_DELAY_POOLS
 294     /* delay_id is already set on original store client */
 295     sc->setDelayId(DelayId::DelayClient(http));
 296 #endif
 297
 298     http->request->lastmod = old_entry->lastmod;
 299
 300     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 301         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 302         if (old_entry->hasEtag(etag) && !etag.weak)
 303             http->request->etag = etag.str;
 304     }
 305
 306     debugs(88, 5, "clientReplyContext::processExpired : lastmod " << entry->lastmod );
 307     http->storeEntry(entry);
 308     assert(http->out.offset == 0);
 309     assert(http->request->clientConnectionManager == http->getConn());
 310
 311     /*
 312      * A refcounted pointer so that FwdState stays around as long as
 313      * this clientReplyContext does
 314      */
 315     Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 316     FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 317
 318     /* Register with storage manager to receive updates when data comes in. */
 319
 320     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 321         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 322
 323     {
 324         /* start counting the length from 0 */
 325         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 326         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 327     }
 328 }
 329
 330 void
 331 clientReplyContext::sendClientUpstreamResponse()
 332 {
 333     StoreIOBuffer tempresult;
 334     removeStoreReference(&old_sc, &old_entry);
 335     /* here the data to send is the data we just received */
 336     tempBuffer.offset = 0;
 337     old_reqsize = 0;
 338     /* sendMoreData tracks the offset as well.
 339      * Force it back to zero */
 340     reqofs = 0;
 341     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 342     /* TODO: provide sendMoreData with the ready parsed reply */
 343     tempresult.length = reqsize;
 344     tempresult.data = tempbuf;
 345     sendMoreData(tempresult);
 346 }
 347
 348 void
 349 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 350 {
 351     clientReplyContext *context = (clientReplyContext *)data;
 352     context->handleIMSReply(result);
 353 }
 354
 355 void
 356 clientReplyContext::sendClientOldEntry()
 357 {
 358     /* Get the old request back */
 359     restoreState();
 360     /* here the data to send is in the next nodes buffers already */
 361     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 362     /* sendMoreData tracks the offset as well.
 363      * Force it back to zero */
 364     reqofs = 0;
 365     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 366     sendMoreData(tempresult);
 367 }
 368
 369 /* This is the workhorse of the HandleIMSReply callback.
 370  *
 371  * It is called when we've got data back from the origin following our
 372  * IMS request to revalidate a stale entry.
 373  */
 374 void
 375 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 376 {
 377     if (deleting)
 378         return;
 379
 380     debugs(88, 3, "handleIMSReply: " << http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes" );
 381
 382     if (http->storeEntry() == NULL)
 383         return;
 384
 385     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 386         return;
 387
 388     /* update size of the request */
 389     reqsize = result.length + reqofs;
 390
 391     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 392
 393     // request to origin was aborted
 394     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 395         debugs(88, 3, "handleIMSReply: request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client" );
 396         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 397         sendClientOldEntry();
 398     }
 399
 400     const HttpReply *old_rep = old_entry->getReply();
 401
 402     // origin replied 304
 403     if (status == Http::scNotModified) {
 404         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 405         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 406
 407         // update headers on existing entry
 408         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 409
 410         // if client sent IMS
 411
 412         if (http->request->flags.ims && !old_entry->modifiedSince(http->request)) {
 413             // forward the 304 from origin
 414             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client");
 415             sendClientUpstreamResponse();
 416         } else {
 417             // send existing entry, it's still valid
 418             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " <<
 419                    old_rep->sline.status() << " to client");
 420             sendClientOldEntry();
 421         }
 422     }
 423
 424     // origin replied with a non-error code
 425     else if (status > Http::scNone && status < Http::scInternalServerError) {
 426         // forward response from origin
 427         http->logType = LOG_TCP_REFRESH_MODIFIED;
 428         debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client");
 429         sendClientUpstreamResponse();
 430     }
 431
 432     // origin replied with an error
 433     else if (http->request->flags.failOnValidationError) {
 434         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 435         debugs(88, 3, "handleIMSReply: origin replied with error " << status <<
 436                ", forwarding to client due to fail_on_validation_err");
 437         sendClientUpstreamResponse();
 438     } else {
 439         // ignore and let client have old entry
 440         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 441         debugs(88, 3, "handleIMSReply: origin replied with error " <<
 442                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 443         sendClientOldEntry();
 444     }
 445 }
 446
 447 SQUIDCEXTERN CSR clientGetMoreData;
 448 SQUIDCEXTERN CSD clientReplyDetach;
 449
 450 /**
 451  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 452  * have been parsed.  Normally this should be a single call, but
 453  * it might take more than one.  As soon as we have the headers,
 454  * we hand off to clientSendMoreData, processExpired, or
 455  * processMiss.
 456  */
 457 void
 458 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 459 {
 460     clientReplyContext *context = (clientReplyContext *)data;
 461     context->cacheHit(result);
 462 }
 463
 464 /**
 465  * Process a possible cache HIT.
 466  */
 467 void
 468 clientReplyContext::cacheHit(StoreIOBuffer result)
 469 {
 470     /** Ignore if the HIT object is being deleted. */
 471     if (deleting) {
 472         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 473         return;
 474     }
 475
 476     StoreEntry *e = http->storeEntry();
 477
 478     HttpRequest *r = http->request;
 479
 480     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 481
 482     if (http->storeEntry() == NULL) {
 483         debugs(88, 3, "clientCacheHit: request aborted");
 484         return;
 485     } else if (result.flags.error) {
 486         /* swap in failure */
 487         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 488         http->logType = LOG_TCP_SWAPFAIL_MISS;
 489         removeClientStoreReference(&sc, http);
 490         processMiss();
 491         return;
 492     }
 493
 494     if (result.length == 0) {
 495         debugs(88, 5, "store IO buffer has no content. MISS");
 496         /* the store couldn't get enough data from the file for us to id the
 497          * object
 498          */
 499         /* treat as a miss */
 500         http->logType = LOG_TCP_MISS;
 501         processMiss();
 502         return;
 503     }
 504
 505     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 506     /* update size of the request */
 507     reqsize = result.length + reqofs;
 508
 509     /*
 510      * Got the headers, now grok them
 511      */
 512     assert(http->logType.oldType == LOG_TCP_HIT);
 513
 514     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 515         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 516         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 517         processMiss();
 518         return;
 519     }
 520
 521     switch (varyEvaluateMatch(e, r)) {
 522
 523     case VARY_NONE:
 524         /* No variance detected. Continue as normal */
 525         break;
 526
 527     case VARY_MATCH:
 528         /* This is the correct entity for this request. Continue */
 529         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 530         break;
 531
 532     case VARY_OTHER:
 533         /* This is not the correct entity for this request. We need
 534          * to requery the cache.
 535          */
 536         removeClientStoreReference(&sc, http);
 537         e = NULL;
 538         /* Note: varyEvalyateMatch updates the request with vary information
 539          * so we only get here once. (it also takes care of cancelling loops)
 540          */
 541         debugs(88, 2, "clientProcessHit: Vary detected!");
 542         clientGetMoreData(ourNode, http);
 543         return;
 544
 545     case VARY_CANCEL:
 546         /* varyEvaluateMatch found a object loop. Process as miss */
 547         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 548         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 549         processMiss();
 550         return;
 551     }
 552
 553     if (r->method == Http::METHOD_PURGE) {
 554         debugs(88, 5, "PURGE gets a HIT");
 555         removeClientStoreReference(&sc, http);
 556         e = NULL;
 557         purgeRequest();
 558         return;
 559     }
 560
 561     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 562         debugs(88, 5, "negative-HIT");
 563         http->logType = LOG_TCP_NEGATIVE_HIT;
 564         sendMoreData(result);
 565     } else if (blockedHit()) {
 566         debugs(88, 5, "send_hit forces a MISS");
 567         http->logType = LOG_TCP_MISS;
 568         processMiss();
 569         return;
 570     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 571         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 572         /*
 573          * We hold a stale copy; it needs to be validated
 574          */
 575         /*
 576          * The 'needValidation' flag is used to prevent forwarding
 577          * loops between siblings.  If our copy of the object is stale,
 578          * then we should probably only use parents for the validation
 579          * request.  Otherwise two siblings could generate a loop if
 580          * both have a stale version of the object.
 581          */
 582         r->flags.needValidation = true;
 583
 584         if (e->lastmod < 0) {
 585             debugs(88, 3, "validate HIT object? NO. Missing Last-Modified header. Do MISS.");
 586             /*
 587              * Previous reply didn't have a Last-Modified header,
 588              * we cannot revalidate it.
 589              */
 590             http->logType = LOG_TCP_MISS;
 591             processMiss();
 592         } else if (r->flags.noCache) {
 593             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 594             /*
 595              * This did not match a refresh pattern that overrides no-cache
 596              * we should honour the client no-cache header.
 597              */
 598             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 599             processMiss();
 600         } else if (r->url.getScheme() == AnyP::PROTO_HTTP) {
 601             debugs(88, 3, "validate HIT object? YES.");
 602             /*
 603              * Object needs to be revalidated
 604              * XXX This could apply to FTP as well, if Last-Modified is known.
 605              */
 606             processExpired();
 607         } else {
 608             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 609             /*
 610              * We don't know how to re-validate other protocols. Handle
 611              * them as if the object has expired.
 612              */
 613             http->logType = LOG_TCP_MISS;
 614             processMiss();
 615         }
 616     } else if (r->conditional()) {
 617         debugs(88, 5, "conditional HIT");
 618         processConditional(result);
 619     } else {
 620         /*
 621          * plain ol' cache hit
 622          */
 623         debugs(88, 5, "plain old HIT");
 624
 625 #if USE_DELAY_POOLS
 626         if (e->store_status != STORE_OK)
 627             http->logType = LOG_TCP_MISS;
 628         else
 629 #endif
 630             if (e->mem_status == IN_MEMORY)
 631                 http->logType = LOG_TCP_MEM_HIT;
 632             else if (Config.onoff.offline)
 633                 http->logType = LOG_TCP_OFFLINE_HIT;
 634
 635         sendMoreData(result);
 636     }
 637 }
 638
 639 /**
 640  * Prepare to fetch the object as it's a cache miss of some kind.
 641  */
 642 void
 643 clientReplyContext::processMiss()
 644 {
 645     char *url = http->uri;
 646     HttpRequest *r = http->request;
 647     ErrorState *err = NULL;
 648     debugs(88, 4, r->method << ' ' << url);
 649
 650     /**
 651      * We might have a left-over StoreEntry from a failed cache hit
 652      * or IMS request.
 653      */
 654     if (http->storeEntry()) {
 655         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 656             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 657             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 658             http->storeEntry()->dump(1);
 659         }
 660
 661         removeClientStoreReference(&sc, http);
 662     }
 663
 664     /** Check if its a PURGE request to be actioned. */
 665     if (r->method == Http::METHOD_PURGE) {
 666         purgeRequest();
 667         return;
 668     }
 669
 670     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 671     if (r->method == Http::METHOD_OTHER) {
 672         purgeAllCached();
 673     }
 674
 675     /** Check if 'only-if-cached' flag is set. Action if so. */
 676     if (http->onlyIfCached()) {
 677         processOnlyIfCachedMiss();
 678         return;
 679     }
 680
 681     /// Deny loops
 682     if (r->flags.loopDetected) {
 683         http->al->http.code = Http::scForbidden;
 684         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 685         createStoreEntry(r->method, RequestFlags());
 686         errorAppendEntry(http->storeEntry(), err);
 687         triggerInitialStoreRead();
 688         return;
 689     } else {
 690         assert(http->out.offset == 0);
 691         createStoreEntry(r->method, r->flags);
 692         triggerInitialStoreRead();
 693
 694         if (http->redirect.status) {
 695             HttpReply *rep = new HttpReply;
 696             http->logType = LOG_TCP_REDIRECT;
 697             http->storeEntry()->releaseRequest();
 698             rep->redirect(http->redirect.status, http->redirect.location);
 699             http->storeEntry()->replaceHttpReply(rep);
 700             http->storeEntry()->complete();
 701             return;
 702         }
 703
 704         assert(r->clientConnectionManager == http->getConn());
 705
 706         /** Start forwarding to get the new object from network */
 707         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 708         FwdState::Start(conn, http->storeEntry(), r, http->al);
 709     }
 710 }
 711
 712 /**
 713  * client issued a request with an only-if-cached cache-control directive;
 714  * we did not find a cached object that can be returned without
 715  *     contacting other servers;
 716  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 717  */
 718 void
 719 clientReplyContext::processOnlyIfCachedMiss()
 720 {
 721     debugs(88, 4, http->request->method << ' ' << http->uri);
 722     http->al->http.code = Http::scGatewayTimeout;
 723     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 724                                        http->getConn()->clientConnection->remote, http->request);
 725     removeClientStoreReference(&sc, http);
 726     startError(err);
 727 }
 728
 729 /// process conditional request from client
 730 void
 731 clientReplyContext::processConditional(StoreIOBuffer &result)
 732 {
 733     StoreEntry *const e = http->storeEntry();
 734
 735     if (e->getReply()->sline.status() != Http::scOkay) {
 736         debugs(88, 4, "clientReplyContext::processConditional: Reply code " <<
 737                e->getReply()->sline.status() << " != 200");
 738         http->logType = LOG_TCP_MISS;
 739         processMiss();
 740         return;
 741     }
 742
 743     HttpRequest &r = *http->request;
 744
 745     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 746         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 747         sendPreconditionFailedError();
 748         return;
 749     }
 750
 751     bool matchedIfNoneMatch = false;
 752     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 753         if (!e->hasIfNoneMatchEtag(r)) {
 754             // RFC 2616: ignore IMS if If-None-Match did not match
 755             r.flags.ims = false;
 756             r.ims = -1;
 757             r.imslen = 0;
 758             r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 759             http->logType = LOG_TCP_MISS;
 760             sendMoreData(result);
 761             return;
 762         }
 763
 764         if (!r.flags.ims) {
 765             // RFC 2616: if If-None-Match matched and there is no IMS,
 766             // reply with 304 Not Modified or 412 Precondition Failed
 767             sendNotModifiedOrPreconditionFailedError();
 768             return;
 769         }
 770
 771         // otherwise check IMS below to decide if we reply with 304 or 412
 772         matchedIfNoneMatch = true;
 773     }
 774
 775     if (r.flags.ims) {
 776         // handle If-Modified-Since requests from the client
 777         if (e->modifiedSince(&r)) {
 778             http->logType = LOG_TCP_IMS_HIT;
 779             sendMoreData(result);
 780             return;
 781         }
 782
 783         if (matchedIfNoneMatch) {
 784             // If-None-Match matched, reply with 304 Not Modified or
 785             // 412 Precondition Failed
 786             sendNotModifiedOrPreconditionFailedError();
 787             return;
 788         }
 789
 790         // otherwise reply with 304 Not Modified
 791         sendNotModified();
 792     }
 793 }
 794
 795 /// whether squid.conf send_hit prevents us from serving this hit
 796 bool
 797 clientReplyContext::blockedHit() const
 798 {
 799     if (!Config.accessList.sendHit)
 800         return false; // hits are not blocked by default
 801
 802     if (http->flags.internal)
 803         return false; // internal content "hits" cannot be blocked
 804
 805     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 806         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 807         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 808         HTTPMSGLOCK(chl->reply);
 809         return chl->fastCheck() != ACCESS_ALLOWED; // when in doubt, block
 810     }
 811
 812     // This does not happen, I hope, because we are called from CacheHit, which
 813     // is called via a storeClientCopy() callback, and store should initialize
 814     // the reply before calling that callback.
 815     debugs(88, 3, "Missing reply!");
 816     return false;
 817 }
 818
 819 void
 820 clientReplyContext::purgeRequestFindObjectToPurge()
 821 {
 822     /* Try to find a base entry */
 823     http->flags.purging = true;
 824     lookingforstore = 1;
 825
 826     // TODO: can we use purgeAllCached() here instead of doing the
 827     // getPublicByRequestMethod() dance?
 828     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 829 }
 830
 831 // Purges all entries with a given url
 832 // TODO: move to SideAgent parent, when we have one
 833 /*
 834  * We probably cannot purge Vary-affected responses because their MD5
 835  * keys depend on vary headers.
 836  */
 837 void
 838 purgeEntriesByUrl(HttpRequest * req, const char *url)
 839 {
 840 #if USE_HTCP
 841     bool get_or_head_sent = false;
 842 #endif
 843
 844     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 845         if (m.respMaybeCacheable()) {
 846             if (StoreEntry *entry = storeGetPublic(url, m)) {
 847                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 848 #if USE_HTCP
 849                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 850                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 851                     get_or_head_sent = true;
 852                 }
 853 #endif
 854                 entry->release();
 855             }
 856         }
 857     }
 858
 859 #if USE_HTCP
 860     if (!get_or_head_sent) {
 861         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 862     }
 863 #endif
 864 }
 865
 866 void
 867 clientReplyContext::purgeAllCached()
 868 {
 869     // XXX: performance regression, c_str() reallocates
 870     SBuf url(http->request->effectiveRequestUri());
 871     purgeEntriesByUrl(http->request, url.c_str());
 872 }
 873
 874 void
 875 clientReplyContext::created(StoreEntry *newEntry)
 876 {
 877     if (lookingforstore == 1)
 878         purgeFoundGet(newEntry);
 879     else if (lookingforstore == 2)
 880         purgeFoundHead(newEntry);
 881     else if (lookingforstore == 3)
 882         purgeDoPurgeGet(newEntry);
 883     else if (lookingforstore == 4)
 884         purgeDoPurgeHead(newEntry);
 885     else if (lookingforstore == 5)
 886         identifyFoundObject(newEntry);
 887 }
 888
 889 void
 890 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 891 {
 892     if (newEntry->isNull()) {
 893         lookingforstore = 2;
 894         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 895     } else
 896         purgeFoundObject (newEntry);
 897 }
 898
 899 void
 900 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 901 {
 902     if (newEntry->isNull())
 903         purgeDoMissPurge();
 904     else
 905         purgeFoundObject (newEntry);
 906 }
 907
 908 void
 909 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 910 {
 911     assert (entry && !entry->isNull());
 912
 913     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 914         http->logType = LOG_TCP_DENIED;
 915         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 916                                            http->getConn()->clientConnection->remote, http->request);
 917         startError(err);
 918         return; // XXX: leaking unused entry if some store does not keep it
 919     }
 920
 921     StoreIOBuffer localTempBuffer;
 922     /* Swap in the metadata */
 923     http->storeEntry(entry);
 924
 925     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 926     http->storeEntry()->createMemObject(storeId(), http->log_uri,
 927                                         http->request->method);
 928
 929     sc = storeClientListAdd(http->storeEntry(), this);
 930
 931     http->logType = LOG_TCP_HIT;
 932
 933     reqofs = 0;
 934
 935     localTempBuffer.offset = http->out.offset;
 936
 937     localTempBuffer.length = next()->readBuffer.length;
 938
 939     localTempBuffer.data = next()->readBuffer.data;
 940
 941     storeClientCopy(sc, http->storeEntry(),
 942                     localTempBuffer, CacheHit, this);
 943 }
 944
 945 void
 946 clientReplyContext::purgeRequest()
 947 {
 948     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 949            Config2.onoff.enable_purge);
 950
 951     if (!Config2.onoff.enable_purge) {
 952         http->logType = LOG_TCP_DENIED;
 953         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 954         startError(err);
 955         return;
 956     }
 957
 958     /* Release both IP cache */
 959     ipcacheInvalidate(http->request->url.host());
 960
 961     if (!http->flags.purging)
 962         purgeRequestFindObjectToPurge();
 963     else
 964         purgeDoMissPurge();
 965 }
 966
 967 void
 968 clientReplyContext::purgeDoMissPurge()
 969 {
 970     http->logType = LOG_TCP_MISS;
 971     lookingforstore = 3;
 972     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
 973 }
 974
 975 void
 976 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
 977 {
 978     assert (newEntry);
 979     /* Move to new() when that is created */
 980     purgeStatus = Http::scNotFound;
 981
 982     if (!newEntry->isNull()) {
 983         /* Release the cached URI */
 984         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
 985 #if USE_HTCP
 986         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
 987 #endif
 988         newEntry->release();
 989         purgeStatus = Http::scOkay;
 990     }
 991
 992     lookingforstore = 4;
 993     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 994 }
 995
 996 void
 997 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
 998 {
 999     if (newEntry && !newEntry->isNull()) {
1000         debugs(88, 4, "HEAD " << newEntry->url());
1001 #if USE_HTCP
1002         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1003 #endif
1004         newEntry->release();
1005         purgeStatus = Http::scOkay;
1006     }
1007
1008     /* And for Vary, release the base URI if none of the headers was included in the request */
1009     if (!http->request->vary_headers.isEmpty()
1010             && http->request->vary_headers.find('=') != SBuf::npos) {
1011         // XXX: performance regression, c_str() reallocates
1012         SBuf tmp(http->request->effectiveRequestUri());
1013         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1014
1015         if (entry) {
1016             debugs(88, 4, "Vary GET " << entry->url());
1017 #if USE_HTCP
1018             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1019 #endif
1020             entry->release();
1021             purgeStatus = Http::scOkay;
1022         }
1023
1024         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1025
1026         if (entry) {
1027             debugs(88, 4, "Vary HEAD " << entry->url());
1028 #if USE_HTCP
1029             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1030 #endif
1031             entry->release();
1032             purgeStatus = Http::scOkay;
1033         }
1034     }
1035
1036     /*
1037      * Make a new entry to hold the reply to be written
1038      * to the client.
1039      */
1040     /* FIXME: This doesn't need to go through the store. Simply
1041      * push down the client chain
1042      */
1043     createStoreEntry(http->request->method, RequestFlags());
1044
1045     triggerInitialStoreRead();
1046
1047     HttpReply *rep = new HttpReply;
1048     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1049     http->storeEntry()->replaceHttpReply(rep);
1050     http->storeEntry()->complete();
1051 }
1052
1053 void
1054 clientReplyContext::traceReply(clientStreamNode * node)
1055 {
1056     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1057     StoreIOBuffer localTempBuffer;
1058     createStoreEntry(http->request->method, RequestFlags());
1059     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1060     localTempBuffer.length = nextNode->readBuffer.length;
1061     localTempBuffer.data = nextNode->readBuffer.data;
1062     storeClientCopy(sc, http->storeEntry(),
1063                     localTempBuffer, SendMoreData, this);
1064     http->storeEntry()->releaseRequest();
1065     http->storeEntry()->buffer();
1066     HttpReply *rep = new HttpReply;
1067     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1068     http->storeEntry()->replaceHttpReply(rep);
1069     http->request->swapOut(http->storeEntry());
1070     http->storeEntry()->complete();
1071 }
1072
1073 #define SENDING_BODY 0
1074 #define SENDING_HDRSONLY 1
1075 int
1076 clientReplyContext::checkTransferDone()
1077 {
1078     StoreEntry *entry = http->storeEntry();
1079
1080     if (entry == NULL)
1081         return 0;
1082
1083     /*
1084      * For now, 'done_copying' is used for special cases like
1085      * Range and HEAD requests.
1086      */
1087     if (http->flags.done_copying)
1088         return 1;
1089
1090     if (http->request->flags.chunkedReply && !flags.complete) {
1091         // last-chunk was not sent
1092         return 0;
1093     }
1094
1095     /*
1096      * Handle STORE_OK objects.
1097      * objectLen(entry) will be set proprely.
1098      * RC: Does objectLen(entry) include the Headers?
1099      * RC: Yes.
1100      */
1101     if (entry->store_status == STORE_OK) {
1102         return storeOKTransferDone();
1103     } else {
1104         return storeNotOKTransferDone();
1105     }
1106 }
1107
1108 int
1109 clientReplyContext::storeOKTransferDone() const
1110 {
1111     assert(http->storeEntry()->objectLen() >= 0);
1112     assert(http->storeEntry()->objectLen() >= headers_sz);
1113     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1114         debugs(88,3,HERE << "storeOKTransferDone " <<
1115                " out.offset=" << http->out.offset <<
1116                " objectLen()=" << http->storeEntry()->objectLen() <<
1117                " headers_sz=" << headers_sz);
1118         return 1;
1119     }
1120
1121     return 0;
1122 }
1123
1124 int
1125 clientReplyContext::storeNotOKTransferDone() const
1126 {
1127     /*
1128      * Now, handle STORE_PENDING objects
1129      */
1130     MemObject *mem = http->storeEntry()->mem_obj;
1131     assert(mem != NULL);
1132     assert(http->request != NULL);
1133     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1134     HttpReply const *curReply = mem->getReply();
1135
1136     if (headers_sz == 0)
1137         /* haven't found end of headers yet */
1138         return 0;
1139
1140     /*
1141      * Figure out how much data we are supposed to send.
1142      * If we are sending a body and we don't have a content-length,
1143      * then we must wait for the object to become STORE_OK.
1144      */
1145     if (curReply->content_length < 0)
1146         return 0;
1147
1148     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1149
1150     if (http->out.size < expectedLength)
1151         return 0;
1152     else {
1153         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1154                " out.size=" << http->out.size <<
1155                " expectedLength=" << expectedLength);
1156         return 1;
1157     }
1158 }
1159
1160 /* A write has completed, what is the next status based on the
1161  * canonical request data?
1162  * 1 something is wrong
1163  * 0 nothing is wrong.
1164  *
1165  */
1166 int
1167 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1168 {
1169 #if SIZEOF_INT64_T == 4
1170     if (http->out.size > 0x7FFF0000) {
1171         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1172         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1173         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1174         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1175         return 1;
1176     }
1177
1178     if (http->out.offset > 0x7FFF0000) {
1179         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1180         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1181         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1182         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1183         return 1;
1184     }
1185
1186 #endif
1187     return 0;
1188 }
1189
1190 /* Preconditions:
1191  * *http is a valid structure.
1192  * fd is either -1, or an open fd.
1193  *
1194  * TODO: enumify this
1195  *
1196  * This function is used by any http request sink, to determine the status
1197  * of the object.
1198  */
1199 clientStream_status_t
1200 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1201 {
1202     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1203     assert (context);
1204     assert (context->http == http);
1205     return context->replyStatus();
1206 }
1207
1208 clientStream_status_t
1209 clientReplyContext::replyStatus()
1210 {
1211     int done;
1212     /* Here because lower nodes don't need it */
1213
1214     if (http->storeEntry() == NULL) {
1215         debugs(88, 5, "clientReplyStatus: no storeEntry");
1216         return STREAM_FAILED;   /* yuck, but what can we do? */
1217     }
1218
1219     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1220         /* TODO: Could upstream read errors (result.flags.error) be
1221          * lost, and result in undersize requests being considered
1222          * complete. Should we tcp reset such connections ?
1223          */
1224         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1225         return STREAM_FAILED;
1226     }
1227
1228     if ((done = checkTransferDone()) != 0 || flags.complete) {
1229         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1230         /* Ok we're finished, but how? */
1231
1232         const int64_t expectedBodySize =
1233             http->storeEntry()->getReply()->bodySize(http->request->method);
1234         if (!http->request->flags.proxyKeepalive && expectedBodySize < 0) {
1235             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
1236             return STREAM_FAILED;
1237         }
1238
1239         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1240             debugs(88, 5, "clientReplyStatus: truncated response body");
1241             return STREAM_UNPLANNED_COMPLETE;
1242         }
1243
1244         if (!done) {
1245             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1246             return STREAM_FAILED;
1247         }
1248
1249         if (expectedBodySize >= 0 && !http->gotEnough()) {
1250             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1251             return STREAM_UNPLANNED_COMPLETE;
1252         }
1253
1254         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1255                http->request->flags.proxyKeepalive);
1256         return STREAM_COMPLETE;
1257     }
1258
1259     // XXX: Should this be checked earlier? We could return above w/o checking.
1260     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1261         /* 4096 is a margin for the HTTP headers included in out.offset */
1262         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1263         return STREAM_FAILED;
1264     }
1265
1266     return STREAM_NONE;
1267 }
1268
1269 /* Responses with no body will not have a content-type header,
1270  * which breaks the rep_mime_type acl, which
1271  * coincidentally, is the most common acl for reply access lists.
1272  * A better long term fix for this is to allow acl matchs on the various
1273  * status codes, and then supply a default ruleset that puts these
1274  * codes before any user defines access entries. That way the user
1275  * can choose to block these responses where appropriate, but won't get
1276  * mysterious breakages.
1277  */
1278 bool
1279 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1280 {
1281     bool result;
1282
1283     switch (sline) {
1284
1285     case Http::scContinue:
1286
1287     case Http::scSwitchingProtocols:
1288
1289     case Http::scProcessing:
1290
1291     case Http::scNoContent:
1292
1293     case Http::scNotModified:
1294         result = true;
1295         break;
1296
1297     default:
1298         result = false;
1299     }
1300
1301     return result;
1302 }
1303
1304 /**
1305  * Generate the reply headers sent to client.
1306  *
1307  * Filters out unwanted entries and hop-by-hop from original reply header
1308  * then adds extra entries if we have more info than origin server
1309  * then adds Squid specific entries
1310  */
1311 void
1312 clientReplyContext::buildReplyHeader()
1313 {
1314     HttpHeader *hdr = &reply->header;
1315     const bool is_hit = http->logType.isTcpHit();
1316     HttpRequest *request = http->request;
1317 #if DONT_FILTER_THESE
1318     /* but you might want to if you run Squid as an HTTP accelerator */
1319     /* hdr->delById(HDR_ACCEPT_RANGES); */
1320     hdr->delById(HDR_ETAG);
1321 #endif
1322
1323     if (is_hit)
1324         hdr->delById(Http::HdrType::SET_COOKIE);
1325     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1326
1327     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1328     // remove the Proxy-Authenticate header
1329     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0))
1330         reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1331
1332     reply->header.removeHopByHopEntries();
1333
1334     //    if (request->range)
1335     //      clientBuildRangeHeader(http, reply);
1336
1337     /*
1338      * Add a estimated Age header on cache hits.
1339      */
1340     if (is_hit) {
1341         /*
1342          * Remove any existing Age header sent by upstream caches
1343          * (note that the existing header is passed along unmodified
1344          * on cache misses)
1345          */
1346         hdr->delById(Http::HdrType::AGE);
1347         /*
1348          * This adds the calculated object age. Note that the details of the
1349          * age calculation is performed by adjusting the timestamp in
1350          * StoreEntry::timestampsSet(), not here.
1351          */
1352         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1353             hdr->delById(Http::HdrType::DATE);
1354             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1355         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1356             // Swap the Date: header to current time if we are simulating an origin
1357             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1358             if (h)
1359                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1360             hdr->delById(Http::HdrType::DATE);
1361             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1362             h = hdr->findEntry(Http::HdrType::EXPIRES);
1363             if (h && http->storeEntry()->expires >= 0) {
1364                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1365                 hdr->delById(Http::HdrType::EXPIRES);
1366                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1367             }
1368             if (http->storeEntry()->timestamp <= squid_curtime) {
1369                 // put X-Cache-Age: instead of Age:
1370                 char age[64];
1371                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1372                 hdr->putExt("X-Cache-Age", age);
1373             }
1374         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1375             hdr->putInt(Http::HdrType::AGE,
1376                         squid_curtime - http->storeEntry()->timestamp);
1377             /* Signal old objects.  NB: rfc 2616 is not clear,
1378              * by implication, on whether we should do this to all
1379              * responses, or only cache hits.
1380              * 14.46 states it ONLY applys for heuristically caclulated
1381              * freshness values, 13.2.4 doesn't specify the same limitation.
1382              * We interpret RFC 2616 under the combination.
1383              */
1384             /* TODO: if maxage or s-maxage is present, don't do this */
1385
1386             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1387                 char tbuf[512];
1388                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1389                           "113", ThisCache,
1390                           "This cache hit is still fresh and more than 1 day old");
1391                 hdr->putStr(Http::HdrType::WARNING, tbuf);
1392             }
1393         }
1394     }
1395
1396     /* RFC 2616: Section 14.18
1397      *
1398      * Add a Date: header if missing.
1399      * We have access to a clock therefore are required to amend any shortcoming in servers.
1400      *
1401      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1402      */
1403     if ( !hdr->has(Http::HdrType::DATE) ) {
1404         if (!http->storeEntry())
1405             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1406         else if (http->storeEntry()->timestamp > 0)
1407             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1408         else {
1409             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1410             /* dump something useful about the problem */
1411             http->storeEntry()->dump(DBG_IMPORTANT);
1412         }
1413     }
1414
1415     // add Warnings required by RFC 2616 if serving a stale hit
1416     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1417         hdr->putWarning(110, "Response is stale");
1418         if (http->request->flags.needValidation)
1419             hdr->putWarning(111, "Revalidation failed");
1420     }
1421
1422     /* Filter unproxyable authentication types */
1423     if (http->logType.oldType != LOG_TCP_DENIED &&
1424             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1425         HttpHeaderPos pos = HttpHeaderInitPos;
1426         HttpHeaderEntry *e;
1427
1428         int connection_auth_blocked = 0;
1429         while ((e = hdr->getEntry(&pos))) {
1430             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1431                 const char *value = e->value.rawBuf();
1432
1433                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1434                         (value[4] == '\0' || value[4] == ' '))
1435                         ||
1436                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1437                          (value[9] == '\0' || value[9] == ' '))
1438                         ||
1439                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1440                          (value[8] == '\0' || value[8] == ' '))) {
1441                     if (request->flags.connectionAuthDisabled) {
1442                         hdr->delAt(pos, connection_auth_blocked);
1443                         continue;
1444                     }
1445                     request->flags.mustKeepalive = true;
1446                     if (!request->flags.accelerated && !request->flags.intercepted) {
1447                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1448                         /*
1449                           We send "Connection: Proxy-Support" header to mark
1450                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1451                           understand the semantics of this header. The RFC should have included
1452                           this recommendation.
1453                         */
1454                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1455                     }
1456                     break;
1457                 }
1458             }
1459         }
1460
1461         if (connection_auth_blocked)
1462             hdr->refreshMask();
1463     }
1464
1465 #if USE_AUTH
1466     /* Handle authentication headers */
1467     if (http->logType.oldType == LOG_TCP_DENIED &&
1468             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1469               reply->sline.status() == Http::scUnauthorized)
1470        ) {
1471         /* Add authentication header */
1472         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1473          * depends on authenticate behaviour: all schemes to date send no extra
1474          * data on 407/401 responses, and do not check the accel state on 401/407
1475          * responses
1476          */
1477         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1478     } else if (request->auth_user_request != NULL)
1479         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1480 #endif
1481
1482     /* Append X-Cache */
1483     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1484                       is_hit ? "HIT" : "MISS", getMyHostname());
1485
1486 #if USE_CACHE_DIGESTS
1487     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1488     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1489                       lookup_type ? lookup_type : "NONE",
1490                       getMyHostname(), getMyPort());
1491
1492 #endif
1493
1494     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1495                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1496                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1497
1498     /* Check whether we should send keep-alive */
1499     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1500         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1501         request->flags.proxyKeepalive = false;
1502     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1503         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1504         request->flags.proxyKeepalive = false;
1505     } else if (request->flags.proxyKeepalive && shutting_down) {
1506         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1507         request->flags.proxyKeepalive = false;
1508     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1509         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1510         request->flags.proxyKeepalive = false;
1511     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1512         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1513         request->flags.proxyKeepalive = false;
1514     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1515         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1516         request->flags.proxyKeepalive = false;
1517     } else if (request->flags.sslBumped && !reply->persistent()) {
1518         // We do not really have to close, but we pretend we are a tunnel.
1519         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1520         request->flags.proxyKeepalive = false;
1521     } else if (request->pinnedConnection() && !reply->persistent()) {
1522         // The peer wants to close the pinned connection
1523         debugs(88, 3, "pinned reply forces close");
1524         request->flags.proxyKeepalive = false;
1525     } else if (http->getConn()) {
1526         ConnStateData * conn = http->getConn();
1527         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1528             // The listening port closed because of a reconfigure
1529             debugs(88, 3, "listening port closed");
1530             request->flags.proxyKeepalive = false;
1531         }
1532     }
1533
1534     // Decide if we send chunked reply
1535     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1536         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1537         request->flags.chunkedReply = true;
1538         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1539     }
1540
1541     /* Append VIA */
1542     if (Config.onoff.via) {
1543         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1544         String strVia;
1545         hdr->getList(Http::HdrType::VIA, &strVia);
1546         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1547                  reply->sline.version.major,
1548                  reply->sline.version.minor,
1549                  ThisCache);
1550         strListAdd(&strVia, bbuf, ',');
1551         hdr->delById(Http::HdrType::VIA);
1552         hdr->putStr(Http::HdrType::VIA, strVia.termedBuf());
1553     }
1554     /* Signal keep-alive or close explicitly */
1555     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1556
1557 #if ADD_X_REQUEST_URI
1558     /*
1559      * Knowing the URI of the request is useful when debugging persistent
1560      * connections in a client; we cannot guarantee the order of http headers,
1561      * but X-Request-URI is likely to be the very last header to ease use from a
1562      * debugger [hdr->entries.count-1].
1563      */
1564     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1565                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1566
1567 #endif
1568
1569     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1570     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1571         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1572             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1573         }
1574         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1575     }
1576
1577     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1578 }
1579
1580 void
1581 clientReplyContext::cloneReply()
1582 {
1583     assert(reply == NULL);
1584
1585     reply = http->storeEntry()->getReply()->clone();
1586     HTTPMSGLOCK(reply);
1587
1588     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1589         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1590         reply->sline.version = Http::ProtocolVersion();
1591     }
1592
1593     /* do header conversions */
1594     buildReplyHeader();
1595 }
1596
1597 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1598 /// We cannot just ignore the entry because it may be locking or otherwise
1599 /// holding an associated cache resource of some sort.
1600 void
1601 clientReplyContext::forgetHit()
1602 {
1603     StoreEntry *e = http->storeEntry();
1604     assert(e); // or we are not dealing with a hit
1605     // We probably have not locked the entry earlier, unfortunately. We lock it
1606     // now so that we can unlock two lines later (and trigger cleanup).
1607     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1608     // used so inconsistently that simply adding locking there leads to bugs.
1609     e->lock("clientReplyContext::forgetHit");
1610     http->storeEntry(NULL);
1611     e->unlock("clientReplyContext::forgetHit"); // may delete e
1612 }
1613
1614 void
1615 clientReplyContext::identifyStoreObject()
1616 {
1617     HttpRequest *r = http->request;
1618
1619     if (r->flags.cachable || r->flags.internal) {
1620         lookingforstore = 5;
1621         StoreEntry::getPublicByRequest (this, r);
1622     } else {
1623         identifyFoundObject (NullStoreEntry::getInstance());
1624     }
1625 }
1626
1627 /**
1628  * Check state of the current StoreEntry object.
1629  * to see if we can determine the final status of the request.
1630  */
1631 void
1632 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1633 {
1634     StoreEntry *e = newEntry;
1635     HttpRequest *r = http->request;
1636
1637     /** \li If the entry received isNull() then we ignore it. */
1638     if (e->isNull()) {
1639         http->storeEntry(NULL);
1640     } else {
1641         http->storeEntry(e);
1642     }
1643
1644     e = http->storeEntry();
1645
1646     /* Release IP-cache entries on reload */
1647     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1648       * 'invalidate' the cached IP entries for this request ???
1649       */
1650     if (r->flags.noCache || r->flags.noCacheHack())
1651         ipcacheInvalidateNegative(r->url.host());
1652
1653 #if USE_CACHE_DIGESTS
1654     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1655 #endif
1656
1657     if (NULL == http->storeEntry()) {
1658         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1659         debugs(85, 3, "StoreEntry is NULL -  MISS");
1660         http->logType = LOG_TCP_MISS;
1661         doGetMoreData();
1662         return;
1663     }
1664
1665     if (Config.onoff.offline) {
1666         /** \li If we are running in offline mode set to HIT */
1667         debugs(85, 3, "offline HIT " << *e);
1668         http->logType = LOG_TCP_HIT;
1669         doGetMoreData();
1670         return;
1671     }
1672
1673     if (http->redirect.status) {
1674         /** \li If redirection status is True force this to be a MISS */
1675         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1676         forgetHit();
1677         http->logType = LOG_TCP_REDIRECT;
1678         doGetMoreData();
1679         return;
1680     }
1681
1682     if (!e->validToSend()) {
1683         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1684         forgetHit();
1685         http->logType = LOG_TCP_MISS;
1686         doGetMoreData();
1687         return;
1688     }
1689
1690     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1691         /* \li Special entries are always hits, no matter what the client says */
1692         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1693         http->logType = LOG_TCP_HIT;
1694         doGetMoreData();
1695         return;
1696     }
1697
1698     if (r->flags.noCache) {
1699         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1700         forgetHit();
1701         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1702         doGetMoreData();
1703         return;
1704     }
1705
1706     debugs(85, 3, "default HIT " << *e);
1707     http->logType = LOG_TCP_HIT;
1708     doGetMoreData();
1709 }
1710
1711 /**
1712  * Request more data from the store for the client Stream
1713  * This is *the* entry point to this module.
1714  *
1715  * Preconditions:
1716  *  - This is the head of the list.
1717  *  - There is at least one more node.
1718  *  - Data context is not null
1719  */
1720 void
1721 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1722 {
1723     /* Test preconditions */
1724     assert(aNode != NULL);
1725     assert(cbdataReferenceValid(aNode));
1726     assert(aNode->node.prev == NULL);
1727     assert(aNode->node.next != NULL);
1728     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1729     assert (context);
1730     assert(context->http == http);
1731
1732     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1733
1734     if (!context->ourNode)
1735         context->ourNode = aNode;
1736
1737     /* no cbdatareference, this is only used once, and safely */
1738     if (context->flags.storelogiccomplete) {
1739         StoreIOBuffer tempBuffer;
1740         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1741         tempBuffer.length = next->readBuffer.length;
1742         tempBuffer.data = next->readBuffer.data;
1743
1744         storeClientCopy(context->sc, http->storeEntry(),
1745                         tempBuffer, clientReplyContext::SendMoreData, context);
1746         return;
1747     }
1748
1749     if (context->http->request->method == Http::METHOD_PURGE) {
1750         context->purgeRequest();
1751         return;
1752     }
1753
1754     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1755
1756     if (context->http->request->method == Http::METHOD_TRACE) {
1757         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1758             context->traceReply(aNode);
1759             return;
1760         }
1761
1762         /* continue forwarding, not finished yet. */
1763         http->logType = LOG_TCP_MISS;
1764
1765         context->doGetMoreData();
1766     } else
1767         context->identifyStoreObject();
1768 }
1769
1770 void
1771 clientReplyContext::doGetMoreData()
1772 {
1773     /* We still have to do store logic processing - vary, cache hit etc */
1774     if (http->storeEntry() != NULL) {
1775         /* someone found the object in the cache for us */
1776         StoreIOBuffer localTempBuffer;
1777
1778         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1779
1780         MemObject *mem_obj = http->storeEntry()->makeMemObject();
1781         if (!mem_obj->hasUris()) {
1782             /*
1783              * This if-block exists because we don't want to clobber
1784              * a preexiting mem_obj->method value if the mem_obj
1785              * already exists.  For example, when a HEAD request
1786              * is a cache hit for a GET response, we want to keep
1787              * the method as GET.
1788              */
1789             mem_obj->setUris(storeId(), http->log_uri, http->request->method);
1790             /**
1791              * Here we can see if the object was
1792              * created using URL or alternative StoreID from helper.
1793              */
1794             debugs(88, 3, "storeId: " << http->storeEntry()->mem_obj->storeId());
1795         }
1796
1797         sc = storeClientListAdd(http->storeEntry(), this);
1798 #if USE_DELAY_POOLS
1799         sc->setDelayId(DelayId::DelayClient(http));
1800 #endif
1801
1802         assert(http->logType.oldType == LOG_TCP_HIT);
1803         reqofs = 0;
1804         /* guarantee nothing has been sent yet! */
1805         assert(http->out.size == 0);
1806         assert(http->out.offset == 0);
1807
1808         if (Ip::Qos::TheConfig.isHitTosActive()) {
1809             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1810         }
1811
1812         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1813             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1814         }
1815
1816         localTempBuffer.offset = reqofs;
1817         localTempBuffer.length = getNextNode()->readBuffer.length;
1818         localTempBuffer.data = getNextNode()->readBuffer.data;
1819         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1820     } else {
1821         /* MISS CASE, http->logType is already set! */
1822         processMiss();
1823     }
1824 }
1825
1826 /** The next node has removed itself from the stream. */
1827 void
1828 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1829 {
1830     /** detach from the stream */
1831     clientStreamDetach(node, http);
1832 }
1833
1834 /**
1835  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1836  * such, writes processed message to the message recipient
1837  */
1838 void
1839 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1840 {
1841     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1842     context->sendMoreData (result);
1843 }
1844
1845 void
1846 clientReplyContext::makeThisHead()
1847 {
1848     /* At least, I think thats what this does */
1849     dlinkDelete(&http->active, &ClientActiveRequests);
1850     dlinkAdd(http, &http->active, &ClientActiveRequests);
1851 }
1852
1853 bool
1854 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1855 {
1856     return /* aborted request */
1857         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1858         /* Upstream read error */ (result.flags.error) ||
1859         /* Upstream EOF */ (sizeToProcess == 0);
1860 }
1861
1862 void
1863 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1864 {
1865     /** call clientWriteComplete so the client socket gets closed
1866      *
1867      * We call into the stream, because we don't know that there is a
1868      * client socket!
1869      */
1870     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1871     StoreIOBuffer localTempBuffer;
1872     flags.complete = 1;
1873     http->request->flags.streamError = true;
1874     localTempBuffer.flags.error = result.flags.error;
1875     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1876                          localTempBuffer);
1877 }
1878
1879 void
1880 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1881 {
1882     StoreIOBuffer localTempBuffer;
1883
1884     if (result.length == 0) {
1885         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1886         flags.complete = 1;
1887     }
1888
1889     assert(result.offset - headers_sz == next()->readBuffer.offset);
1890     localTempBuffer.offset = result.offset - headers_sz;
1891     localTempBuffer.length = result.length;
1892
1893     if (localTempBuffer.length)
1894         localTempBuffer.data = source;
1895
1896     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1897                          localTempBuffer);
1898 }
1899
1900 clientStreamNode *
1901 clientReplyContext::next() const
1902 {
1903     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1904     return getNextNode();
1905 }
1906
1907 void
1908 clientReplyContext::sendBodyTooLargeError()
1909 {
1910     Ip::Address tmp_noaddr;
1911     tmp_noaddr.setNoAddr(); // TODO: make a global const
1912     http->logType = LOG_TCP_DENIED_REPLY;
1913     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1914                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1915                                        http->request);
1916     removeClientStoreReference(&(sc), http);
1917     HTTPMSGUNLOCK(reply);
1918     startError(err);
1919
1920 }
1921
1922 /// send 412 (Precondition Failed) to client
1923 void
1924 clientReplyContext::sendPreconditionFailedError()
1925 {
1926     http->logType = LOG_TCP_HIT;
1927     ErrorState *const err =
1928         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1929                          NULL, http->getConn()->clientConnection->remote, http->request);
1930     removeClientStoreReference(&sc, http);
1931     HTTPMSGUNLOCK(reply);
1932     startError(err);
1933 }
1934
1935 /// send 304 (Not Modified) to client
1936 void
1937 clientReplyContext::sendNotModified()
1938 {
1939     StoreEntry *e = http->storeEntry();
1940     const time_t timestamp = e->timestamp;
1941     HttpReply *const temprep = e->getReply()->make304();
1942     http->logType = LOG_TCP_IMS_HIT;
1943     removeClientStoreReference(&sc, http);
1944     createStoreEntry(http->request->method, RequestFlags());
1945     e = http->storeEntry();
1946     // Copy timestamp from the original entry so the 304
1947     // reply has a meaningful Age: header.
1948     e->timestampsSet();
1949     e->timestamp = timestamp;
1950     e->replaceHttpReply(temprep);
1951     e->complete();
1952     /*
1953      * TODO: why put this in the store and then serialise it and
1954      * then parse it again. Simply mark the request complete in
1955      * our context and write the reply struct to the client side.
1956      */
1957     triggerInitialStoreRead();
1958 }
1959
1960 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1961 /// depending on request method
1962 void
1963 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1964 {
1965     if (http->request->method == Http::METHOD_GET ||
1966             http->request->method == Http::METHOD_HEAD)
1967         sendNotModified();
1968     else
1969         sendPreconditionFailedError();
1970 }
1971
1972 void
1973 clientReplyContext::processReplyAccess ()
1974 {
1975     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
1976     assert(reply);
1977
1978     /** Don't block our own responses or HTTP status messages */
1979     if (http->logType.oldType == LOG_TCP_DENIED ||
1980             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
1981             alwaysAllowResponse(reply->sline.status())) {
1982         headers_sz = reply->hdr_sz;
1983         processReplyAccessResult(ACCESS_ALLOWED);
1984         return;
1985     }
1986
1987     /** Check for reply to big error */
1988     if (reply->expectedBodyTooLarge(*http->request)) {
1989         sendBodyTooLargeError();
1990         return;
1991     }
1992
1993     headers_sz = reply->hdr_sz;
1994
1995     /** check for absent access controls (permit by default) */
1996     if (!Config.accessList.reply) {
1997         processReplyAccessResult(ACCESS_ALLOWED);
1998         return;
1999     }
2000
2001     /** Process http_reply_access lists */
2002     ACLFilledChecklist *replyChecklist =
2003         clientAclChecklistCreate(Config.accessList.reply, http);
2004     replyChecklist->reply = reply;
2005     HTTPMSGLOCK(replyChecklist->reply);
2006     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2007 }
2008
2009 void
2010 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2011 {
2012     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2013     me->processReplyAccessResult(rv);
2014 }
2015
2016 void
2017 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2018 {
2019     debugs(88, 2, "The reply for " << http->request->method
2020            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2021            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2022
2023     if (accessAllowed != ACCESS_ALLOWED) {
2024         ErrorState *err;
2025         err_type page_id;
2026         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2027
2028         http->logType = LOG_TCP_DENIED_REPLY;
2029
2030         if (page_id == ERR_NONE)
2031             page_id = ERR_ACCESS_DENIED;
2032
2033         Ip::Address tmp_noaddr;
2034         tmp_noaddr.setNoAddr();
2035         err = clientBuildError(page_id, Http::scForbidden, NULL,
2036                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2037                                http->request);
2038
2039         removeClientStoreReference(&sc, http);
2040
2041         HTTPMSGUNLOCK(reply);
2042
2043         startError(err);
2044
2045         return;
2046     }
2047
2048     /* Ok, the reply is allowed, */
2049     http->loggingEntry(http->storeEntry());
2050
2051     ssize_t body_size = reqofs - reply->hdr_sz;
2052     if (body_size < 0) {
2053         reqofs = reply->hdr_sz;
2054         body_size = 0;
2055     }
2056
2057     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2058            (int) body_size << " bytes after " << reply->hdr_sz <<
2059            " bytes of headers");
2060
2061 #if USE_SQUID_ESI
2062
2063     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2064             !alwaysAllowResponse(reply->sline.status()) &&
2065             esiEnableProcessing(reply)) {
2066         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2067         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2068                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2069     }
2070
2071 #endif
2072
2073     if (http->request->method == Http::METHOD_HEAD) {
2074         /* do not forward body for HEAD replies */
2075         body_size = 0;
2076         http->flags.done_copying = true;
2077         flags.complete = 1;
2078     }
2079
2080     assert (!flags.headersSent);
2081     flags.headersSent = true;
2082
2083     StoreIOBuffer localTempBuffer;
2084     char *buf = next()->readBuffer.data;
2085     char *body_buf = buf + reply->hdr_sz;
2086
2087     //Server side may disable ranges under some circumstances.
2088
2089     if ((!http->request->range))
2090         next()->readBuffer.offset = 0;
2091
2092     body_buf -= next()->readBuffer.offset;
2093
2094     if (next()->readBuffer.offset != 0) {
2095         if (next()->readBuffer.offset > body_size) {
2096             /* Can't use any of the body we received. send nothing */
2097             localTempBuffer.length = 0;
2098             localTempBuffer.data = NULL;
2099         } else {
2100             localTempBuffer.length = body_size - next()->readBuffer.offset;
2101             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2102         }
2103     } else {
2104         localTempBuffer.length = body_size;
2105         localTempBuffer.data = body_buf;
2106     }
2107
2108     /* TODO??: move the data in the buffer back by the request header size */
2109     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2110                          http, reply, localTempBuffer);
2111
2112     return;
2113 }
2114
2115 void
2116 clientReplyContext::sendMoreData (StoreIOBuffer result)
2117 {
2118     if (deleting)
2119         return;
2120
2121     StoreEntry *entry = http->storeEntry();
2122
2123     ConnStateData * conn = http->getConn();
2124
2125     // too late, our conn is closing
2126     // TODO: should we also quit?
2127     if (conn == NULL) {
2128         debugs(33,3, "not sending more data to a closed connection" );
2129         return;
2130     }
2131     if (!conn->isOpen()) {
2132         debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2133         return;
2134     }
2135     if (conn->pinning.zeroReply) {
2136         debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2137         return;
2138     }
2139
2140     char *buf = next()->readBuffer.data;
2141
2142     if (buf != result.data) {
2143         /* we've got to copy some data */
2144         assert(result.length <= next()->readBuffer.length);
2145         memcpy(buf, result.data, result.length);
2146     }
2147
2148     if (reqofs==0 && !http->logType.isTcpHit() && Comm::IsConnOpen(conn->clientConnection)) {
2149         if (Ip::Qos::TheConfig.isHitTosActive()) {
2150             Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2151         }
2152         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2153             Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2154         }
2155     }
2156
2157     /* We've got the final data to start pushing... */
2158     flags.storelogiccomplete = 1;
2159
2160     reqofs += result.length;
2161
2162     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2163
2164     assert(http->request != NULL);
2165
2166     /* ESI TODO: remove this assert once everything is stable */
2167     assert(http->client_stream.head->data
2168            && cbdataReferenceValid(http->client_stream.head->data));
2169
2170     makeThisHead();
2171
2172     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2173            reqofs << " bytes (" << result.length <<
2174            " new bytes)");
2175     debugs(88, 5, "clientReplyContext::sendMoreData:"
2176            << conn->clientConnection <<
2177            " '" << entry->url() << "'" <<
2178            " out.offset=" << http->out.offset);
2179
2180     /* update size of the request */
2181     reqsize = reqofs;
2182
2183     if (errorInStream(result, reqofs)) {
2184         sendStreamError(result);
2185         return;
2186     }
2187
2188     if (flags.headersSent) {
2189         pushStreamData (result, buf);
2190         return;
2191     }
2192
2193     cloneReply();
2194
2195 #if USE_DELAY_POOLS
2196     if (sc)
2197         sc->setDelayId(DelayId::DelayClient(http,reply));
2198 #endif
2199
2200     /* handle headers */
2201
2202     if (Config.onoff.log_mime_hdrs) {
2203         size_t k;
2204
2205         if ((k = headersEnd(buf, reqofs))) {
2206             safe_free(http->al->headers.reply);
2207             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2208             xstrncpy(http->al->headers.reply, buf, k);
2209         }
2210     }
2211
2212     holdingBuffer = result;
2213     processReplyAccess();
2214     return;
2215 }
2216
2217 /* Using this breaks the client layering just a little!
2218  */
2219 void
2220 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2221 {
2222     assert(http != NULL);
2223     /*
2224      * For erroneous requests, we might not have a h->request,
2225      * so make a fake one.
2226      */
2227
2228     if (http->request == NULL) {
2229         http->request = new HttpRequest(m, AnyP::PROTO_NONE, null_string);
2230         HTTPMSGLOCK(http->request);
2231     }
2232
2233     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2234
2235     // Make entry collapsable ASAP, to increase collapsing chances for others,
2236     // TODO: every must-revalidate and similar request MUST reach the origin,
2237     // but do we have to prohibit others from collapsing on that request?
2238     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2239             !reqFlags.needValidation &&
2240             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2241         // make the entry available for future requests now
2242         Store::Root().allowCollapsing(e, reqFlags, m);
2243     }
2244
2245     sc = storeClientListAdd(e, this);
2246
2247 #if USE_DELAY_POOLS
2248     sc->setDelayId(DelayId::DelayClient(http));
2249 #endif
2250
2251     reqofs = 0;
2252
2253     reqsize = 0;
2254
2255     /* I don't think this is actually needed! -- adrian */
2256     /* http->reqbuf = http->norm_reqbuf; */
2257     //    assert(http->reqbuf == http->norm_reqbuf);
2258     /* The next line is illegal because we don't know if the client stream
2259      * buffers have been set up
2260      */
2261     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2262     //        SendMoreData, this);
2263     /* So, we mark the store logic as complete */
2264     flags.storelogiccomplete = 1;
2265
2266     /* and get the caller to request a read, from whereever they are */
2267     /* NOTE: after ANY data flows down the pipe, even one step,
2268      * this function CAN NOT be used to manage errors
2269      */
2270     http->storeEntry(e);
2271 }
2272
2273 ErrorState *
2274 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2275                  Ip::Address &src_addr, HttpRequest * request)
2276 {
2277     ErrorState *err = new ErrorState(page_id, status, request);
2278     err->src_addr = src_addr;
2279
2280     if (url)
2281         err->url = xstrdup(url);
2282
2283     return err;
2284 }
2285