src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "globals.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47 #if USE_SQUID_ESI
  48 #include "esi/Esi.h"
  49 #endif
  50
  51 #include <memory>
  52
  53 CBDATA_CLASS_INIT(clientReplyContext);
  54
  55 /* Local functions */
  56 extern "C" CSS clientReplyStatus;
  57 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  58
  59 /* privates */
  60
  61 clientReplyContext::~clientReplyContext()
  62 {
  63     deleting = true;
  64     /* This may trigger a callback back into SendMoreData as the cbdata
  65      * is still valid
  66      */
  67     removeClientStoreReference(&sc, http);
  68     /* old_entry might still be set if we didn't yet get the reply
  69      * code in HandleIMSReply() */
  70     removeStoreReference(&old_sc, &old_entry);
  71     safe_free(tempBuffer.data);
  72     cbdataReferenceDone(http);
  73     HTTPMSGUNLOCK(reply);
  74 }
  75
  76 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) : http (cbdataReference(clientContext)), old_entry (NULL), old_sc(NULL), deleting(false)
  77 {}
  78
  79 /** Create an error in the store awaiting the client side to read it.
  80  *
  81  * This may be better placed in the clientStream logic, but it has not been
  82  * relocated there yet
  83  */
  84 void
  85 clientReplyContext::setReplyToError(
  86     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
  87     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
  88 #if USE_AUTH
  89     Auth::UserRequest::Pointer auth_user_request
  90 #else
  91     void*
  92 #endif
  93 )
  94 {
  95     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
  96
  97     if (unparsedrequest)
  98         errstate->request_hdrs = xstrdup(unparsedrequest);
  99
 100 #if USE_AUTH
 101     errstate->auth_user_request = auth_user_request;
 102 #endif
 103     setReplyToError(method, errstate);
 104 }
 105
 106 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 107 {
 108     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 109         /* prevent confusion over whether we default to persistent or not */
 110         http->request->flags.proxyKeepalive = false;
 111
 112     http->al->http.code = errstate->httpStatus;
 113
 114     if (http->request)
 115         http->request->ignoreRange("responding with a Squid-generated error");
 116
 117     createStoreEntry(method, RequestFlags());
 118     assert(errstate->callback_data == NULL);
 119     errorAppendEntry(http->storeEntry(), errstate);
 120     /* Now the caller reads to get this */
 121 }
 122
 123 void
 124 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 125 {
 126     Must(futureReply);
 127     http->al->http.code = futureReply->sline.status();
 128
 129     HttpRequestMethod method;
 130     if (http->request) { // nil on responses to unparsable requests
 131         http->request->ignoreRange("responding with a Squid-generated reply");
 132         method = http->request->method;
 133     }
 134
 135     createStoreEntry(method, RequestFlags());
 136
 137     http->storeEntry()->storeErrorResponse(futureReply);
 138     /* Now the caller reads to get futureReply */
 139 }
 140
 141 // Assumes that the entry contains an error response without Content-Range.
 142 // To use with regular entries, make HTTP Range header removal conditional.
 143 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 144 {
 145     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 146     sc = storeClientListAdd(entry, this);
 147 #if USE_DELAY_POOLS
 148     sc->setDelayId(DelayId::DelayClient(http));
 149 #endif
 150     reqofs = 0;
 151     reqsize = 0;
 152     if (http->request)
 153         http->request->ignoreRange(reason);
 154     flags.storelogiccomplete = 1;
 155     http->storeEntry(entry);
 156 }
 157
 158 void
 159 clientReplyContext::removeStoreReference(store_client ** scp,
 160         StoreEntry ** ep)
 161 {
 162     StoreEntry *e;
 163     store_client *sc_tmp = *scp;
 164
 165     if ((e = *ep) != NULL) {
 166         *ep = NULL;
 167         storeUnregister(sc_tmp, e, this);
 168         *scp = NULL;
 169         e->unlock("clientReplyContext::removeStoreReference");
 170     }
 171 }
 172
 173 void
 174 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 175 {
 176     StoreEntry *reference = aHttpRequest->storeEntry();
 177     removeStoreReference(scp, &reference);
 178     aHttpRequest->storeEntry(reference);
 179 }
 180
 181 void
 182 clientReplyContext::saveState()
 183 {
 184     assert(old_sc == NULL);
 185     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 186     old_entry = http->storeEntry();
 187     old_sc = sc;
 188     old_reqsize = reqsize;
 189     tempBuffer.offset = reqofs;
 190     /* Prevent accessing the now saved entries */
 191     http->storeEntry(NULL);
 192     sc = NULL;
 193     reqsize = 0;
 194     reqofs = 0;
 195 }
 196
 197 void
 198 clientReplyContext::restoreState()
 199 {
 200     assert(old_sc != NULL);
 201     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 202     removeClientStoreReference(&sc, http);
 203     http->storeEntry(old_entry);
 204     sc = old_sc;
 205     reqsize = old_reqsize;
 206     reqofs = tempBuffer.offset;
 207     /* Prevent accessed the old saved entries */
 208     old_entry = NULL;
 209     old_sc = NULL;
 210     old_reqsize = 0;
 211     tempBuffer.offset = 0;
 212 }
 213
 214 void
 215 clientReplyContext::startError(ErrorState * err)
 216 {
 217     createStoreEntry(http->request->method, RequestFlags());
 218     triggerInitialStoreRead();
 219     errorAppendEntry(http->storeEntry(), err);
 220 }
 221
 222 clientStreamNode *
 223 clientReplyContext::getNextNode() const
 224 {
 225     return (clientStreamNode *)ourNode->node.next->data;
 226 }
 227
 228 /* This function is wrong - the client parameters don't include the
 229  * header offset
 230  */
 231 void
 232 clientReplyContext::triggerInitialStoreRead()
 233 {
 234     /* when confident, 0 becomes reqofs, and then this factors into
 235      * startSendProcess
 236      */
 237     assert(reqofs == 0);
 238     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 239     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 240 }
 241
 242 /* there is an expired entry in the store.
 243  * setup a temporary buffer area and perform an IMS to the origin
 244  */
 245 void
 246 clientReplyContext::processExpired()
 247 {
 248     const char *url = storeId();
 249     StoreEntry *entry = NULL;
 250     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 251     assert(http->storeEntry()->lastmod >= 0);
 252     /*
 253      * check if we are allowed to contact other servers
 254      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 255      *      a stale entry *if* it matches client requirements
 256      */
 257
 258     if (http->onlyIfCached()) {
 259         processOnlyIfCachedMiss();
 260         return;
 261     }
 262
 263     http->request->flags.refresh = true;
 264 #if STORE_CLIENT_LIST_DEBUG
 265     /* Prevent a race with the store client memory free routines
 266      */
 267     assert(storeClientIsThisAClient(sc, this));
 268 #endif
 269     /* Prepare to make a new temporary request */
 270     saveState();
 271     entry = storeCreateEntry(url,
 272                              http->log_uri, http->request->flags, http->request->method);
 273     /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 274     sc = storeClientListAdd(entry, this);
 275 #if USE_DELAY_POOLS
 276     /* delay_id is already set on original store client */
 277     sc->setDelayId(DelayId::DelayClient(http));
 278 #endif
 279
 280     http->request->lastmod = old_entry->lastmod;
 281
 282     if (!http->request->header.has(HDR_IF_NONE_MATCH)) {
 283         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 284         if (old_entry->hasEtag(etag) && !etag.weak)
 285             http->request->etag = etag.str;
 286     }
 287
 288     debugs(88, 5, "clientReplyContext::processExpired : lastmod " << entry->lastmod );
 289     http->storeEntry(entry);
 290     assert(http->out.offset == 0);
 291     assert(http->request->clientConnectionManager == http->getConn());
 292
 293     /*
 294      * A refcounted pointer so that FwdState stays around as long as
 295      * this clientReplyContext does
 296      */
 297     Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 298     FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 299
 300     /* Register with storage manager to receive updates when data comes in. */
 301
 302     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 303         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 304
 305     {
 306         /* start counting the length from 0 */
 307         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 308         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 309     }
 310 }
 311
 312 void
 313 clientReplyContext::sendClientUpstreamResponse()
 314 {
 315     StoreIOBuffer tempresult;
 316     removeStoreReference(&old_sc, &old_entry);
 317     /* here the data to send is the data we just received */
 318     tempBuffer.offset = 0;
 319     old_reqsize = 0;
 320     /* sendMoreData tracks the offset as well.
 321      * Force it back to zero */
 322     reqofs = 0;
 323     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 324     /* TODO: provide sendMoreData with the ready parsed reply */
 325     tempresult.length = reqsize;
 326     tempresult.data = tempbuf;
 327     sendMoreData(tempresult);
 328 }
 329
 330 void
 331 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 332 {
 333     clientReplyContext *context = (clientReplyContext *)data;
 334     context->handleIMSReply(result);
 335 }
 336
 337 void
 338 clientReplyContext::sendClientOldEntry()
 339 {
 340     /* Get the old request back */
 341     restoreState();
 342     /* here the data to send is in the next nodes buffers already */
 343     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 344     /* sendMoreData tracks the offset as well.
 345      * Force it back to zero */
 346     reqofs = 0;
 347     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 348     sendMoreData(tempresult);
 349 }
 350
 351 /* This is the workhorse of the HandleIMSReply callback.
 352  *
 353  * It is called when we've got data back from the origin following our
 354  * IMS request to revalidate a stale entry.
 355  */
 356 void
 357 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 358 {
 359     if (deleting)
 360         return;
 361
 362     debugs(88, 3, "handleIMSReply: " << http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes" );
 363
 364     if (http->storeEntry() == NULL)
 365         return;
 366
 367     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 368         return;
 369
 370     /* update size of the request */
 371     reqsize = result.length + reqofs;
 372
 373     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 374
 375     // request to origin was aborted
 376     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 377         debugs(88, 3, "handleIMSReply: request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client" );
 378         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 379         sendClientOldEntry();
 380     }
 381
 382     HttpReply *old_rep = (HttpReply *) old_entry->getReply();
 383
 384     // origin replied 304
 385     if (status == Http::scNotModified) {
 386         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 387         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 388
 389         // update headers on existing entry
 390         old_rep->updateOnNotModified(http->storeEntry()->getReply());
 391         old_entry->timestampsSet();
 392
 393         // if client sent IMS
 394
 395         if (http->request->flags.ims && !old_entry->modifiedSince(http->request)) {
 396             // forward the 304 from origin
 397             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client");
 398             sendClientUpstreamResponse();
 399         } else {
 400             // send existing entry, it's still valid
 401             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " <<
 402                    old_rep->sline.status() << " to client");
 403             sendClientOldEntry();
 404         }
 405     }
 406
 407     // origin replied with a non-error code
 408     else if (status > Http::scNone && status < Http::scInternalServerError) {
 409         // forward response from origin
 410         http->logType = LOG_TCP_REFRESH_MODIFIED;
 411         debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client");
 412         sendClientUpstreamResponse();
 413     }
 414
 415     // origin replied with an error
 416     else if (http->request->flags.failOnValidationError) {
 417         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 418         debugs(88, 3, "handleIMSReply: origin replied with error " << status <<
 419                ", forwarding to client due to fail_on_validation_err");
 420         sendClientUpstreamResponse();
 421     } else {
 422         // ignore and let client have old entry
 423         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 424         debugs(88, 3, "handleIMSReply: origin replied with error " <<
 425                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 426         sendClientOldEntry();
 427     }
 428 }
 429
 430 SQUIDCEXTERN CSR clientGetMoreData;
 431 SQUIDCEXTERN CSD clientReplyDetach;
 432
 433 /**
 434  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 435  * have been parsed.  Normally this should be a single call, but
 436  * it might take more than one.  As soon as we have the headers,
 437  * we hand off to clientSendMoreData, processExpired, or
 438  * processMiss.
 439  */
 440 void
 441 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 442 {
 443     clientReplyContext *context = (clientReplyContext *)data;
 444     context->cacheHit(result);
 445 }
 446
 447 /**
 448  * Process a possible cache HIT.
 449  */
 450 void
 451 clientReplyContext::cacheHit(StoreIOBuffer result)
 452 {
 453     /** Ignore if the HIT object is being deleted. */
 454     if (deleting) {
 455         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 456         return;
 457     }
 458
 459     StoreEntry *e = http->storeEntry();
 460
 461     HttpRequest *r = http->request;
 462
 463     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 464
 465     if (http->storeEntry() == NULL) {
 466         debugs(88, 3, "clientCacheHit: request aborted");
 467         return;
 468     } else if (result.flags.error) {
 469         /* swap in failure */
 470         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 471         http->logType = LOG_TCP_SWAPFAIL_MISS;
 472         removeClientStoreReference(&sc, http);
 473         processMiss();
 474         return;
 475     }
 476
 477     if (result.length == 0) {
 478         debugs(88, 5, "store IO buffer has no content. MISS");
 479         /* the store couldn't get enough data from the file for us to id the
 480          * object
 481          */
 482         /* treat as a miss */
 483         http->logType = LOG_TCP_MISS;
 484         processMiss();
 485         return;
 486     }
 487
 488     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 489     /* update size of the request */
 490     reqsize = result.length + reqofs;
 491
 492     /*
 493      * Got the headers, now grok them
 494      */
 495     assert(http->logType == LOG_TCP_HIT);
 496
 497     if (strcmp(e->mem_obj->storeId(), http->request->storeId()) != 0) {
 498         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 499         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 500         processMiss();
 501         return;
 502     }
 503
 504     switch (varyEvaluateMatch(e, r)) {
 505
 506     case VARY_NONE:
 507         /* No variance detected. Continue as normal */
 508         break;
 509
 510     case VARY_MATCH:
 511         /* This is the correct entity for this request. Continue */
 512         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 513         break;
 514
 515     case VARY_OTHER:
 516         /* This is not the correct entity for this request. We need
 517          * to requery the cache.
 518          */
 519         removeClientStoreReference(&sc, http);
 520         e = NULL;
 521         /* Note: varyEvalyateMatch updates the request with vary information
 522          * so we only get here once. (it also takes care of cancelling loops)
 523          */
 524         debugs(88, 2, "clientProcessHit: Vary detected!");
 525         clientGetMoreData(ourNode, http);
 526         return;
 527
 528     case VARY_CANCEL:
 529         /* varyEvaluateMatch found a object loop. Process as miss */
 530         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 531         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 532         processMiss();
 533         return;
 534     }
 535
 536     if (r->method == Http::METHOD_PURGE) {
 537         debugs(88, 5, "PURGE gets a HIT");
 538         removeClientStoreReference(&sc, http);
 539         e = NULL;
 540         purgeRequest();
 541         return;
 542     }
 543
 544     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 545         debugs(88, 5, "negative-HIT");
 546         http->logType = LOG_TCP_NEGATIVE_HIT;
 547         sendMoreData(result);
 548     } else if (blockedHit()) {
 549         debugs(88, 5, "send_hit forces a MISS");
 550         http->logType = LOG_TCP_MISS;
 551         processMiss();
 552         return;
 553     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 554         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 555         /*
 556          * We hold a stale copy; it needs to be validated
 557          */
 558         /*
 559          * The 'needValidation' flag is used to prevent forwarding
 560          * loops between siblings.  If our copy of the object is stale,
 561          * then we should probably only use parents for the validation
 562          * request.  Otherwise two siblings could generate a loop if
 563          * both have a stale version of the object.
 564          */
 565         r->flags.needValidation = true;
 566
 567         if (e->lastmod < 0) {
 568             debugs(88, 3, "validate HIT object? NO. Missing Last-Modified header. Do MISS.");
 569             /*
 570              * Previous reply didn't have a Last-Modified header,
 571              * we cannot revalidate it.
 572              */
 573             http->logType = LOG_TCP_MISS;
 574             processMiss();
 575         } else if (r->flags.noCache) {
 576             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 577             /*
 578              * This did not match a refresh pattern that overrides no-cache
 579              * we should honour the client no-cache header.
 580              */
 581             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 582             processMiss();
 583         } else if (r->url.getScheme() == AnyP::PROTO_HTTP) {
 584             debugs(88, 3, "validate HIT object? YES.");
 585             /*
 586              * Object needs to be revalidated
 587              * XXX This could apply to FTP as well, if Last-Modified is known.
 588              */
 589             processExpired();
 590         } else {
 591             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 592             /*
 593              * We don't know how to re-validate other protocols. Handle
 594              * them as if the object has expired.
 595              */
 596             http->logType = LOG_TCP_MISS;
 597             processMiss();
 598         }
 599     } else if (r->conditional()) {
 600         debugs(88, 5, "conditional HIT");
 601         processConditional(result);
 602     } else {
 603         /*
 604          * plain ol' cache hit
 605          */
 606         debugs(88, 5, "plain old HIT");
 607
 608 #if USE_DELAY_POOLS
 609         if (e->store_status != STORE_OK)
 610             http->logType = LOG_TCP_MISS;
 611         else
 612 #endif
 613             if (e->mem_status == IN_MEMORY)
 614                 http->logType = LOG_TCP_MEM_HIT;
 615             else if (Config.onoff.offline)
 616                 http->logType = LOG_TCP_OFFLINE_HIT;
 617
 618         sendMoreData(result);
 619     }
 620 }
 621
 622 /**
 623  * Prepare to fetch the object as it's a cache miss of some kind.
 624  */
 625 void
 626 clientReplyContext::processMiss()
 627 {
 628     char *url = http->uri;
 629     HttpRequest *r = http->request;
 630     ErrorState *err = NULL;
 631     debugs(88, 4, r->method << ' ' << url);
 632
 633     /**
 634      * We might have a left-over StoreEntry from a failed cache hit
 635      * or IMS request.
 636      */
 637     if (http->storeEntry()) {
 638         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 639             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 640             debugs(88, DBG_CRITICAL, "\tlog_type = " << LogTags_str[http->logType]);
 641             http->storeEntry()->dump(1);
 642         }
 643
 644         removeClientStoreReference(&sc, http);
 645     }
 646
 647     /** Check if its a PURGE request to be actioned. */
 648     if (r->method == Http::METHOD_PURGE) {
 649         purgeRequest();
 650         return;
 651     }
 652
 653     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 654     if (r->method == Http::METHOD_OTHER) {
 655         purgeAllCached();
 656     }
 657
 658     /** Check if 'only-if-cached' flag is set. Action if so. */
 659     if (http->onlyIfCached()) {
 660         processOnlyIfCachedMiss();
 661         return;
 662     }
 663
 664     /// Deny loops
 665     if (r->flags.loopDetected) {
 666         http->al->http.code = Http::scForbidden;
 667         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 668         createStoreEntry(r->method, RequestFlags());
 669         errorAppendEntry(http->storeEntry(), err);
 670         triggerInitialStoreRead();
 671         return;
 672     } else {
 673         assert(http->out.offset == 0);
 674         createStoreEntry(r->method, r->flags);
 675         triggerInitialStoreRead();
 676
 677         if (http->redirect.status) {
 678             HttpReply *rep = new HttpReply;
 679             http->logType = LOG_TCP_REDIRECT;
 680             http->storeEntry()->releaseRequest();
 681             rep->redirect(http->redirect.status, http->redirect.location);
 682             http->storeEntry()->replaceHttpReply(rep);
 683             http->storeEntry()->complete();
 684             return;
 685         }
 686
 687         assert(r->clientConnectionManager == http->getConn());
 688
 689         /** Start forwarding to get the new object from network */
 690         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 691         FwdState::Start(conn, http->storeEntry(), r, http->al);
 692     }
 693 }
 694
 695 /**
 696  * client issued a request with an only-if-cached cache-control directive;
 697  * we did not find a cached object that can be returned without
 698  *     contacting other servers;
 699  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 700  */
 701 void
 702 clientReplyContext::processOnlyIfCachedMiss()
 703 {
 704     debugs(88, 4, http->request->method << ' ' << http->uri);
 705     http->al->http.code = Http::scGatewayTimeout;
 706     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 707                                        http->getConn()->clientConnection->remote, http->request);
 708     removeClientStoreReference(&sc, http);
 709     startError(err);
 710 }
 711
 712 /// process conditional request from client
 713 void
 714 clientReplyContext::processConditional(StoreIOBuffer &result)
 715 {
 716     StoreEntry *const e = http->storeEntry();
 717
 718     if (e->getReply()->sline.status() != Http::scOkay) {
 719         debugs(88, 4, "clientReplyContext::processConditional: Reply code " <<
 720                e->getReply()->sline.status() << " != 200");
 721         http->logType = LOG_TCP_MISS;
 722         processMiss();
 723         return;
 724     }
 725
 726     HttpRequest &r = *http->request;
 727
 728     if (r.header.has(HDR_IF_MATCH) && !e->hasIfMatchEtag(r)) {
 729         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 730         sendPreconditionFailedError();
 731         return;
 732     }
 733
 734     bool matchedIfNoneMatch = false;
 735     if (r.header.has(HDR_IF_NONE_MATCH)) {
 736         if (!e->hasIfNoneMatchEtag(r)) {
 737             // RFC 2616: ignore IMS if If-None-Match did not match
 738             r.flags.ims = false;
 739             r.ims = -1;
 740             r.imslen = 0;
 741             r.header.delById(HDR_IF_MODIFIED_SINCE);
 742             http->logType = LOG_TCP_MISS;
 743             sendMoreData(result);
 744             return;
 745         }
 746
 747         if (!r.flags.ims) {
 748             // RFC 2616: if If-None-Match matched and there is no IMS,
 749             // reply with 304 Not Modified or 412 Precondition Failed
 750             sendNotModifiedOrPreconditionFailedError();
 751             return;
 752         }
 753
 754         // otherwise check IMS below to decide if we reply with 304 or 412
 755         matchedIfNoneMatch = true;
 756     }
 757
 758     if (r.flags.ims) {
 759         // handle If-Modified-Since requests from the client
 760         if (e->modifiedSince(&r)) {
 761             http->logType = LOG_TCP_IMS_HIT;
 762             sendMoreData(result);
 763             return;
 764         }
 765
 766         if (matchedIfNoneMatch) {
 767             // If-None-Match matched, reply with 304 Not Modified or
 768             // 412 Precondition Failed
 769             sendNotModifiedOrPreconditionFailedError();
 770             return;
 771         }
 772
 773         // otherwise reply with 304 Not Modified
 774         sendNotModified();
 775     }
 776 }
 777
 778 /// whether squid.conf send_hit prevents us from serving this hit
 779 bool
 780 clientReplyContext::blockedHit() const
 781 {
 782     if (!Config.accessList.sendHit)
 783         return false; // hits are not blocked by default
 784
 785     if (http->flags.internal)
 786         return false; // internal content "hits" cannot be blocked
 787
 788     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 789         std::auto_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 790         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 791         HTTPMSGLOCK(chl->reply);
 792         return chl->fastCheck() != ACCESS_ALLOWED; // when in doubt, block
 793     }
 794
 795     // This does not happen, I hope, because we are called from CacheHit, which
 796     // is called via a storeClientCopy() callback, and store should initialize
 797     // the reply before calling that callback.
 798     debugs(88, 3, "Missing reply!");
 799     return false;
 800 }
 801
 802 void
 803 clientReplyContext::purgeRequestFindObjectToPurge()
 804 {
 805     /* Try to find a base entry */
 806     http->flags.purging = true;
 807     lookingforstore = 1;
 808
 809     // TODO: can we use purgeAllCached() here instead of doing the
 810     // getPublicByRequestMethod() dance?
 811     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 812 }
 813
 814 // Purges all entries with a given url
 815 // TODO: move to SideAgent parent, when we have one
 816 /*
 817  * We probably cannot purge Vary-affected responses because their MD5
 818  * keys depend on vary headers.
 819  */
 820 void
 821 purgeEntriesByUrl(HttpRequest * req, const char *url)
 822 {
 823 #if USE_HTCP
 824     bool get_or_head_sent = false;
 825 #endif
 826
 827     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 828         if (m.respMaybeCacheable()) {
 829             if (StoreEntry *entry = storeGetPublic(url, m)) {
 830                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 831 #if USE_HTCP
 832                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 833                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 834                     get_or_head_sent = true;
 835                 }
 836 #endif
 837                 entry->release();
 838             }
 839         }
 840     }
 841
 842 #if USE_HTCP
 843     if (!get_or_head_sent) {
 844         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 845     }
 846 #endif
 847 }
 848
 849 void
 850 clientReplyContext::purgeAllCached()
 851 {
 852     const char *url = urlCanonical(http->request);
 853     purgeEntriesByUrl(http->request, url);
 854 }
 855
 856 void
 857 clientReplyContext::created(StoreEntry *newEntry)
 858 {
 859     if (lookingforstore == 1)
 860         purgeFoundGet(newEntry);
 861     else if (lookingforstore == 2)
 862         purgeFoundHead(newEntry);
 863     else if (lookingforstore == 3)
 864         purgeDoPurgeGet(newEntry);
 865     else if (lookingforstore == 4)
 866         purgeDoPurgeHead(newEntry);
 867     else if (lookingforstore == 5)
 868         identifyFoundObject(newEntry);
 869 }
 870
 871 void
 872 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 873 {
 874     if (newEntry->isNull()) {
 875         lookingforstore = 2;
 876         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 877     } else
 878         purgeFoundObject (newEntry);
 879 }
 880
 881 void
 882 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 883 {
 884     if (newEntry->isNull())
 885         purgeDoMissPurge();
 886     else
 887         purgeFoundObject (newEntry);
 888 }
 889
 890 void
 891 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 892 {
 893     assert (entry && !entry->isNull());
 894
 895     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 896         http->logType = LOG_TCP_DENIED;
 897         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 898                                            http->getConn()->clientConnection->remote, http->request);
 899         startError(err);
 900         return; // XXX: leaking unused entry if some store does not keep it
 901     }
 902
 903     StoreIOBuffer localTempBuffer;
 904     /* Swap in the metadata */
 905     http->storeEntry(entry);
 906
 907     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 908     http->storeEntry()->createMemObject(storeId(), http->log_uri,
 909                                         http->request->method);
 910
 911     sc = storeClientListAdd(http->storeEntry(), this);
 912
 913     http->logType = LOG_TCP_HIT;
 914
 915     reqofs = 0;
 916
 917     localTempBuffer.offset = http->out.offset;
 918
 919     localTempBuffer.length = next()->readBuffer.length;
 920
 921     localTempBuffer.data = next()->readBuffer.data;
 922
 923     storeClientCopy(sc, http->storeEntry(),
 924                     localTempBuffer, CacheHit, this);
 925 }
 926
 927 void
 928 clientReplyContext::purgeRequest()
 929 {
 930     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 931            Config2.onoff.enable_purge);
 932
 933     if (!Config2.onoff.enable_purge) {
 934         http->logType = LOG_TCP_DENIED;
 935         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 936         startError(err);
 937         return;
 938     }
 939
 940     /* Release both IP cache */
 941     ipcacheInvalidate(http->request->GetHost());
 942
 943     if (!http->flags.purging)
 944         purgeRequestFindObjectToPurge();
 945     else
 946         purgeDoMissPurge();
 947 }
 948
 949 void
 950 clientReplyContext::purgeDoMissPurge()
 951 {
 952     http->logType = LOG_TCP_MISS;
 953     lookingforstore = 3;
 954     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
 955 }
 956
 957 void
 958 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
 959 {
 960     assert (newEntry);
 961     /* Move to new() when that is created */
 962     purgeStatus = Http::scNotFound;
 963
 964     if (!newEntry->isNull()) {
 965         /* Release the cached URI */
 966         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
 967 #if USE_HTCP
 968         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
 969 #endif
 970         newEntry->release();
 971         purgeStatus = Http::scOkay;
 972     }
 973
 974     lookingforstore = 4;
 975     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 976 }
 977
 978 void
 979 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
 980 {
 981     if (newEntry && !newEntry->isNull()) {
 982         debugs(88, 4, "clientPurgeRequest: HEAD '" << newEntry->url() << "'" );
 983 #if USE_HTCP
 984         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
 985 #endif
 986         newEntry->release();
 987         purgeStatus = Http::scOkay;
 988     }
 989
 990     /* And for Vary, release the base URI if none of the headers was included in the request */
 991
 992     if (http->request->vary_headers
 993             && !strstr(http->request->vary_headers, "=")) {
 994         StoreEntry *entry = storeGetPublic(urlCanonical(http->request), Http::METHOD_GET);
 995
 996         if (entry) {
 997             debugs(88, 4, "clientPurgeRequest: Vary GET '" << entry->url() << "'" );
 998 #if USE_HTCP
 999             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1000 #endif
1001             entry->release();
1002             purgeStatus = Http::scOkay;
1003         }
1004
1005         entry = storeGetPublic(urlCanonical(http->request), Http::METHOD_HEAD);
1006
1007         if (entry) {
1008             debugs(88, 4, "clientPurgeRequest: Vary HEAD '" << entry->url() << "'" );
1009 #if USE_HTCP
1010             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1011 #endif
1012             entry->release();
1013             purgeStatus = Http::scOkay;
1014         }
1015     }
1016
1017     /*
1018      * Make a new entry to hold the reply to be written
1019      * to the client.
1020      */
1021     /* FIXME: This doesn't need to go through the store. Simply
1022      * push down the client chain
1023      */
1024     createStoreEntry(http->request->method, RequestFlags());
1025
1026     triggerInitialStoreRead();
1027
1028     HttpReply *rep = new HttpReply;
1029     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1030     http->storeEntry()->replaceHttpReply(rep);
1031     http->storeEntry()->complete();
1032 }
1033
1034 void
1035 clientReplyContext::traceReply(clientStreamNode * node)
1036 {
1037     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1038     StoreIOBuffer localTempBuffer;
1039     createStoreEntry(http->request->method, RequestFlags());
1040     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1041     localTempBuffer.length = nextNode->readBuffer.length;
1042     localTempBuffer.data = nextNode->readBuffer.data;
1043     storeClientCopy(sc, http->storeEntry(),
1044                     localTempBuffer, SendMoreData, this);
1045     http->storeEntry()->releaseRequest();
1046     http->storeEntry()->buffer();
1047     HttpReply *rep = new HttpReply;
1048     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1049     http->storeEntry()->replaceHttpReply(rep);
1050     http->request->swapOut(http->storeEntry());
1051     http->storeEntry()->complete();
1052 }
1053
1054 #define SENDING_BODY 0
1055 #define SENDING_HDRSONLY 1
1056 int
1057 clientReplyContext::checkTransferDone()
1058 {
1059     StoreEntry *entry = http->storeEntry();
1060
1061     if (entry == NULL)
1062         return 0;
1063
1064     /*
1065      * For now, 'done_copying' is used for special cases like
1066      * Range and HEAD requests.
1067      */
1068     if (http->flags.done_copying)
1069         return 1;
1070
1071     if (http->request->flags.chunkedReply && !flags.complete) {
1072         // last-chunk was not sent
1073         return 0;
1074     }
1075
1076     /*
1077      * Handle STORE_OK objects.
1078      * objectLen(entry) will be set proprely.
1079      * RC: Does objectLen(entry) include the Headers?
1080      * RC: Yes.
1081      */
1082     if (entry->store_status == STORE_OK) {
1083         return storeOKTransferDone();
1084     } else {
1085         return storeNotOKTransferDone();
1086     }
1087 }
1088
1089 int
1090 clientReplyContext::storeOKTransferDone() const
1091 {
1092     assert(http->storeEntry()->objectLen() >= 0);
1093     assert(http->storeEntry()->objectLen() >= headers_sz);
1094     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1095         debugs(88,3,HERE << "storeOKTransferDone " <<
1096                " out.offset=" << http->out.offset <<
1097                " objectLen()=" << http->storeEntry()->objectLen() <<
1098                " headers_sz=" << headers_sz);
1099         return 1;
1100     }
1101
1102     return 0;
1103 }
1104
1105 int
1106 clientReplyContext::storeNotOKTransferDone() const
1107 {
1108     /*
1109      * Now, handle STORE_PENDING objects
1110      */
1111     MemObject *mem = http->storeEntry()->mem_obj;
1112     assert(mem != NULL);
1113     assert(http->request != NULL);
1114     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1115     HttpReply const *curReply = mem->getReply();
1116
1117     if (headers_sz == 0)
1118         /* haven't found end of headers yet */
1119         return 0;
1120
1121     /*
1122      * Figure out how much data we are supposed to send.
1123      * If we are sending a body and we don't have a content-length,
1124      * then we must wait for the object to become STORE_OK.
1125      */
1126     if (curReply->content_length < 0)
1127         return 0;
1128
1129     int64_t expectedLength = curReply->content_length + http->out.headers_sz;
1130
1131     if (http->out.size < expectedLength)
1132         return 0;
1133     else {
1134         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1135                " out.size=" << http->out.size <<
1136                " expectedLength=" << expectedLength);
1137         return 1;
1138     }
1139 }
1140
1141 /* A write has completed, what is the next status based on the
1142  * canonical request data?
1143  * 1 something is wrong
1144  * 0 nothing is wrong.
1145  *
1146  */
1147 int
1148 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1149 {
1150 #if SIZEOF_INT64_T == 4
1151     if (http->out.size > 0x7FFF0000) {
1152         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1153         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1154         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1155         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1156         return 1;
1157     }
1158
1159     if (http->out.offset > 0x7FFF0000) {
1160         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1161         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1162         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1163         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1164         return 1;
1165     }
1166
1167 #endif
1168     return 0;
1169 }
1170
1171 /* Preconditions:
1172  * *http is a valid structure.
1173  * fd is either -1, or an open fd.
1174  *
1175  * TODO: enumify this
1176  *
1177  * This function is used by any http request sink, to determine the status
1178  * of the object.
1179  */
1180 clientStream_status_t
1181 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1182 {
1183     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1184     assert (context);
1185     assert (context->http == http);
1186     return context->replyStatus();
1187 }
1188
1189 clientStream_status_t
1190 clientReplyContext::replyStatus()
1191 {
1192     int done;
1193     /* Here because lower nodes don't need it */
1194
1195     if (http->storeEntry() == NULL) {
1196         debugs(88, 5, "clientReplyStatus: no storeEntry");
1197         return STREAM_FAILED;   /* yuck, but what can we do? */
1198     }
1199
1200     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1201         /* TODO: Could upstream read errors (result.flags.error) be
1202          * lost, and result in undersize requests being considered
1203          * complete. Should we tcp reset such connections ?
1204          */
1205         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1206         return STREAM_FAILED;
1207     }
1208
1209     if ((done = checkTransferDone()) != 0 || flags.complete) {
1210         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1211         /* Ok we're finished, but how? */
1212
1213         const int64_t expectedBodySize =
1214             http->storeEntry()->getReply()->bodySize(http->request->method);
1215         if (!http->request->flags.proxyKeepalive && expectedBodySize < 0) {
1216             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
1217             return STREAM_FAILED;
1218         }
1219
1220         if (!done) {
1221             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1222             return STREAM_FAILED;
1223         }
1224
1225         if (expectedBodySize >= 0 && !http->gotEnough()) {
1226             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1227             return STREAM_UNPLANNED_COMPLETE;
1228         }
1229
1230         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1231                http->request->flags.proxyKeepalive);
1232         return STREAM_COMPLETE;
1233     }
1234
1235     // XXX: Should this be checked earlier? We could return above w/o checking.
1236     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1237         /* 4096 is a margin for the HTTP headers included in out.offset */
1238         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1239         return STREAM_FAILED;
1240     }
1241
1242     return STREAM_NONE;
1243 }
1244
1245 /* Responses with no body will not have a content-type header,
1246  * which breaks the rep_mime_type acl, which
1247  * coincidentally, is the most common acl for reply access lists.
1248  * A better long term fix for this is to allow acl matchs on the various
1249  * status codes, and then supply a default ruleset that puts these
1250  * codes before any user defines access entries. That way the user
1251  * can choose to block these responses where appropriate, but won't get
1252  * mysterious breakages.
1253  */
1254 bool
1255 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1256 {
1257     bool result;
1258
1259     switch (sline) {
1260
1261     case Http::scContinue:
1262
1263     case Http::scSwitchingProtocols:
1264
1265     case Http::scProcessing:
1266
1267     case Http::scNoContent:
1268
1269     case Http::scNotModified:
1270         result = true;
1271         break;
1272
1273     default:
1274         result = false;
1275     }
1276
1277     return result;
1278 }
1279
1280 /**
1281  * Generate the reply headers sent to client.
1282  *
1283  * Filters out unwanted entries and hop-by-hop from original reply header
1284  * then adds extra entries if we have more info than origin server
1285  * then adds Squid specific entries
1286  */
1287 void
1288 clientReplyContext::buildReplyHeader()
1289 {
1290     HttpHeader *hdr = &reply->header;
1291     int is_hit = logTypeIsATcpHit(http->logType);
1292     HttpRequest *request = http->request;
1293 #if DONT_FILTER_THESE
1294     /* but you might want to if you run Squid as an HTTP accelerator */
1295     /* hdr->delById(HDR_ACCEPT_RANGES); */
1296     hdr->delById(HDR_ETAG);
1297 #endif
1298
1299     if (is_hit)
1300         hdr->delById(HDR_SET_COOKIE);
1301     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1302
1303     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1304     // remove the Proxy-Authenticate header
1305     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0))
1306         reply->header.delById(HDR_PROXY_AUTHENTICATE);
1307
1308     reply->header.removeHopByHopEntries();
1309
1310     //    if (request->range)
1311     //      clientBuildRangeHeader(http, reply);
1312
1313     /*
1314      * Add a estimated Age header on cache hits.
1315      */
1316     if (is_hit) {
1317         /*
1318          * Remove any existing Age header sent by upstream caches
1319          * (note that the existing header is passed along unmodified
1320          * on cache misses)
1321          */
1322         hdr->delById(HDR_AGE);
1323         /*
1324          * This adds the calculated object age. Note that the details of the
1325          * age calculation is performed by adjusting the timestamp in
1326          * StoreEntry::timestampsSet(), not here.
1327          */
1328         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1329             hdr->delById(HDR_DATE);
1330             hdr->insertTime(HDR_DATE, squid_curtime);
1331         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1332             // Swap the Date: header to current time if we are simulating an origin
1333             HttpHeaderEntry *h = hdr->findEntry(HDR_DATE);
1334             if (h)
1335                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1336             hdr->delById(HDR_DATE);
1337             hdr->insertTime(HDR_DATE, squid_curtime);
1338             h = hdr->findEntry(HDR_EXPIRES);
1339             if (h && http->storeEntry()->expires >= 0) {
1340                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1341                 hdr->delById(HDR_EXPIRES);
1342                 hdr->insertTime(HDR_EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1343             }
1344             if (http->storeEntry()->timestamp <= squid_curtime) {
1345                 // put X-Cache-Age: instead of Age:
1346                 char age[64];
1347                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1348                 hdr->putExt("X-Cache-Age", age);
1349             }
1350         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1351             hdr->putInt(HDR_AGE,
1352                         squid_curtime - http->storeEntry()->timestamp);
1353             /* Signal old objects.  NB: rfc 2616 is not clear,
1354              * by implication, on whether we should do this to all
1355              * responses, or only cache hits.
1356              * 14.46 states it ONLY applys for heuristically caclulated
1357              * freshness values, 13.2.4 doesn't specify the same limitation.
1358              * We interpret RFC 2616 under the combination.
1359              */
1360             /* TODO: if maxage or s-maxage is present, don't do this */
1361
1362             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1363                 char tbuf[512];
1364                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1365                           "113", ThisCache,
1366                           "This cache hit is still fresh and more than 1 day old");
1367                 hdr->putStr(HDR_WARNING, tbuf);
1368             }
1369         }
1370     }
1371
1372     /* RFC 2616: Section 14.18
1373      *
1374      * Add a Date: header if missing.
1375      * We have access to a clock therefore are required to amend any shortcoming in servers.
1376      *
1377      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1378      */
1379     if ( !hdr->has(HDR_DATE) ) {
1380         if (!http->storeEntry())
1381             hdr->insertTime(HDR_DATE, squid_curtime);
1382         else if (http->storeEntry()->timestamp > 0)
1383             hdr->insertTime(HDR_DATE, http->storeEntry()->timestamp);
1384         else {
1385             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1386             /* dump something useful about the problem */
1387             http->storeEntry()->dump(DBG_IMPORTANT);
1388         }
1389     }
1390
1391     // add Warnings required by RFC 2616 if serving a stale hit
1392     if (http->request->flags.staleIfHit && logTypeIsATcpHit(http->logType)) {
1393         hdr->putWarning(110, "Response is stale");
1394         if (http->request->flags.needValidation)
1395             hdr->putWarning(111, "Revalidation failed");
1396     }
1397
1398     /* Filter unproxyable authentication types */
1399     if (http->logType != LOG_TCP_DENIED &&
1400             hdr->has(HDR_WWW_AUTHENTICATE)) {
1401         HttpHeaderPos pos = HttpHeaderInitPos;
1402         HttpHeaderEntry *e;
1403
1404         int connection_auth_blocked = 0;
1405         while ((e = hdr->getEntry(&pos))) {
1406             if (e->id == HDR_WWW_AUTHENTICATE) {
1407                 const char *value = e->value.rawBuf();
1408
1409                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1410                         (value[4] == '\0' || value[4] == ' '))
1411                         ||
1412                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1413                          (value[9] == '\0' || value[9] == ' '))
1414                         ||
1415                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1416                          (value[8] == '\0' || value[8] == ' '))) {
1417                     if (request->flags.connectionAuthDisabled) {
1418                         hdr->delAt(pos, connection_auth_blocked);
1419                         continue;
1420                     }
1421                     request->flags.mustKeepalive = true;
1422                     if (!request->flags.accelerated && !request->flags.intercepted) {
1423                         httpHeaderPutStrf(hdr, HDR_PROXY_SUPPORT, "Session-Based-Authentication");
1424                         /*
1425                           We send "Connection: Proxy-Support" header to mark
1426                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1427                           understand the semantics of this header. The RFC should have included
1428                           this recommendation.
1429                         */
1430                         httpHeaderPutStrf(hdr, HDR_CONNECTION, "Proxy-support");
1431                     }
1432                     break;
1433                 }
1434             }
1435         }
1436
1437         if (connection_auth_blocked)
1438             hdr->refreshMask();
1439     }
1440
1441 #if USE_AUTH
1442     /* Handle authentication headers */
1443     if (http->logType == LOG_TCP_DENIED &&
1444             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1445               reply->sline.status() == Http::scUnauthorized)
1446        ) {
1447         /* Add authentication header */
1448         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1449          * depends on authenticate behaviour: all schemes to date send no extra
1450          * data on 407/401 responses, and do not check the accel state on 401/407
1451          * responses
1452          */
1453         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1454     } else if (request->auth_user_request != NULL)
1455         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1456 #endif
1457
1458     /* Append X-Cache */
1459     httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s",
1460                       is_hit ? "HIT" : "MISS", getMyHostname());
1461
1462 #if USE_CACHE_DIGESTS
1463     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1464     httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d",
1465                       lookup_type ? lookup_type : "NONE",
1466                       getMyHostname(), getMyPort());
1467
1468 #endif
1469
1470     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1471                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1472                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1473
1474     /* Check whether we should send keep-alive */
1475     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1476         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1477         request->flags.proxyKeepalive = false;
1478     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1479         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1480         request->flags.proxyKeepalive = false;
1481     } else if (request->flags.proxyKeepalive && shutting_down) {
1482         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1483         request->flags.proxyKeepalive = false;
1484     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1485         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1486         request->flags.proxyKeepalive = false;
1487     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1488         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1489         request->flags.proxyKeepalive = false;
1490     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1491         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1492         request->flags.proxyKeepalive = false;
1493     } else if (request->flags.sslBumped && !reply->persistent()) {
1494         // We do not really have to close, but we pretend we are a tunnel.
1495         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1496         request->flags.proxyKeepalive = false;
1497     } else if (request->pinnedConnection() && !reply->persistent()) {
1498         // The peer wants to close the pinned connection
1499         debugs(88, 3, "pinned reply forces close");
1500         request->flags.proxyKeepalive = false;
1501     } else if (http->getConn()) {
1502         ConnStateData * conn = http->getConn();
1503         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1504             // The listening port closed because of a reconfigure
1505             debugs(88, 3, "listening port closed");
1506             request->flags.proxyKeepalive = false;
1507         } else if (Config.Timeout.pconnLifetime && conn->clientConnection->lifeTime() > Config.Timeout.pconnLifetime && conn->getConcurrentRequestCount() <= 1) {
1508             // The persistent connection lifetime exceeded and we are the last parsed request
1509             debugs(88, 3, "persistent connection lifetime exceeded");
1510             request->flags.proxyKeepalive = false;
1511         }
1512     }
1513
1514     // Decide if we send chunked reply
1515     if (maySendChunkedReply &&
1516             request->flags.proxyKeepalive &&
1517             reply->bodySize(request->method) < 0) {
1518         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1519         request->flags.chunkedReply = true;
1520         hdr->putStr(HDR_TRANSFER_ENCODING, "chunked");
1521     }
1522
1523     /* Append VIA */
1524     if (Config.onoff.via) {
1525         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1526         String strVia;
1527         hdr->getList(HDR_VIA, &strVia);
1528         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1529                  reply->sline.version.major,
1530                  reply->sline.version.minor,
1531                  ThisCache);
1532         strListAdd(&strVia, bbuf, ',');
1533         hdr->delById(HDR_VIA);
1534         hdr->putStr(HDR_VIA, strVia.termedBuf());
1535     }
1536     /* Signal keep-alive or close explicitly */
1537     hdr->putStr(HDR_CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1538
1539 #if ADD_X_REQUEST_URI
1540     /*
1541      * Knowing the URI of the request is useful when debugging persistent
1542      * connections in a client; we cannot guarantee the order of http headers,
1543      * but X-Request-URI is likely to be the very last header to ease use from a
1544      * debugger [hdr->entries.count-1].
1545      */
1546     hdr->putStr(HDR_X_REQUEST_URI,
1547                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1548
1549 #endif
1550
1551     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1552     if ( hdr->has(HDR_SURROGATE_CONTROL) ) {
1553         if (!request->header.has(HDR_SURROGATE_CAPABILITY)) {
1554             hdr->delById(HDR_SURROGATE_CONTROL);
1555         }
1556         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1557     }
1558
1559     httpHdrMangleList(hdr, request, ROR_REPLY);
1560 }
1561
1562 void
1563 clientReplyContext::cloneReply()
1564 {
1565     assert(reply == NULL);
1566
1567     reply = http->storeEntry()->getReply()->clone();
1568     HTTPMSGLOCK(reply);
1569
1570     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1571         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1572         reply->sline.version = Http::ProtocolVersion();
1573     }
1574
1575     /* do header conversions */
1576     buildReplyHeader();
1577 }
1578
1579 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1580 /// We cannot just ignore the entry because it may be locking or otherwise
1581 /// holding an associated cache resource of some sort.
1582 void
1583 clientReplyContext::forgetHit()
1584 {
1585     StoreEntry *e = http->storeEntry();
1586     assert(e); // or we are not dealing with a hit
1587     // We probably have not locked the entry earlier, unfortunately. We lock it
1588     // now so that we can unlock two lines later (and trigger cleanup).
1589     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1590     // used so inconsistently that simply adding locking there leads to bugs.
1591     e->lock("clientReplyContext::forgetHit");
1592     http->storeEntry(NULL);
1593     e->unlock("clientReplyContext::forgetHit"); // may delete e
1594 }
1595
1596 void
1597 clientReplyContext::identifyStoreObject()
1598 {
1599     HttpRequest *r = http->request;
1600
1601     if (r->flags.cachable || r->flags.internal) {
1602         lookingforstore = 5;
1603         StoreEntry::getPublicByRequest (this, r);
1604     } else {
1605         identifyFoundObject (NullStoreEntry::getInstance());
1606     }
1607 }
1608
1609 /**
1610  * Check state of the current StoreEntry object.
1611  * to see if we can determine the final status of the request.
1612  */
1613 void
1614 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1615 {
1616     StoreEntry *e = newEntry;
1617     HttpRequest *r = http->request;
1618
1619     /** \li If the entry received isNull() then we ignore it. */
1620     if (e->isNull()) {
1621         http->storeEntry(NULL);
1622     } else {
1623         http->storeEntry(e);
1624     }
1625
1626     e = http->storeEntry();
1627
1628     /* Release IP-cache entries on reload */
1629     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1630       * 'invalidate' the cached IP entries for this request ???
1631       */
1632     if (r->flags.noCache || r->flags.noCacheHack())
1633         ipcacheInvalidateNegative(r->GetHost());
1634
1635 #if USE_CACHE_DIGESTS
1636     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1637 #endif
1638
1639     if (NULL == http->storeEntry()) {
1640         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1641         debugs(85, 3, "StoreEntry is NULL -  MISS");
1642         http->logType = LOG_TCP_MISS;
1643         doGetMoreData();
1644         return;
1645     }
1646
1647     if (Config.onoff.offline) {
1648         /** \li If we are running in offline mode set to HIT */
1649         debugs(85, 3, "offline HIT " << *e);
1650         http->logType = LOG_TCP_HIT;
1651         doGetMoreData();
1652         return;
1653     }
1654
1655     if (http->redirect.status) {
1656         /** \li If redirection status is True force this to be a MISS */
1657         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1658         forgetHit();
1659         http->logType = LOG_TCP_REDIRECT;
1660         doGetMoreData();
1661         return;
1662     }
1663
1664     if (!e->validToSend()) {
1665         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1666         forgetHit();
1667         http->logType = LOG_TCP_MISS;
1668         doGetMoreData();
1669         return;
1670     }
1671
1672     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1673         /* \li Special entries are always hits, no matter what the client says */
1674         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1675         http->logType = LOG_TCP_HIT;
1676         doGetMoreData();
1677         return;
1678     }
1679
1680     if (r->flags.noCache) {
1681         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1682         forgetHit();
1683         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1684         doGetMoreData();
1685         return;
1686     }
1687
1688     debugs(85, 3, "default HIT " << *e);
1689     http->logType = LOG_TCP_HIT;
1690     doGetMoreData();
1691 }
1692
1693 /**
1694  * Request more data from the store for the client Stream
1695  * This is *the* entry point to this module.
1696  *
1697  * Preconditions:
1698  *  - This is the head of the list.
1699  *  - There is at least one more node.
1700  *  - Data context is not null
1701  */
1702 void
1703 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1704 {
1705     /* Test preconditions */
1706     assert(aNode != NULL);
1707     assert(cbdataReferenceValid(aNode));
1708     assert(aNode->node.prev == NULL);
1709     assert(aNode->node.next != NULL);
1710     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1711     assert (context);
1712     assert(context->http == http);
1713
1714     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1715
1716     if (!context->ourNode)
1717         context->ourNode = aNode;
1718
1719     /* no cbdatareference, this is only used once, and safely */
1720     if (context->flags.storelogiccomplete) {
1721         StoreIOBuffer tempBuffer;
1722         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1723         tempBuffer.length = next->readBuffer.length;
1724         tempBuffer.data = next->readBuffer.data;
1725
1726         storeClientCopy(context->sc, http->storeEntry(),
1727                         tempBuffer, clientReplyContext::SendMoreData, context);
1728         return;
1729     }
1730
1731     if (context->http->request->method == Http::METHOD_PURGE) {
1732         context->purgeRequest();
1733         return;
1734     }
1735
1736     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1737
1738     if (context->http->request->method == Http::METHOD_TRACE) {
1739         if (context->http->request->header.getInt64(HDR_MAX_FORWARDS) == 0) {
1740             context->traceReply(aNode);
1741             return;
1742         }
1743
1744         /* continue forwarding, not finished yet. */
1745         http->logType = LOG_TCP_MISS;
1746
1747         context->doGetMoreData();
1748     } else
1749         context->identifyStoreObject();
1750 }
1751
1752 void
1753 clientReplyContext::doGetMoreData()
1754 {
1755     /* We still have to do store logic processing - vary, cache hit etc */
1756     if (http->storeEntry() != NULL) {
1757         /* someone found the object in the cache for us */
1758         StoreIOBuffer localTempBuffer;
1759
1760         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1761
1762         MemObject *mem_obj = http->storeEntry()->makeMemObject();
1763         if (!mem_obj->hasUris()) {
1764             /*
1765              * This if-block exists because we don't want to clobber
1766              * a preexiting mem_obj->method value if the mem_obj
1767              * already exists.  For example, when a HEAD request
1768              * is a cache hit for a GET response, we want to keep
1769              * the method as GET.
1770              */
1771             mem_obj->setUris(storeId(), http->log_uri, http->request->method);
1772             /**
1773              * Here we can see if the object was
1774              * created using URL or alternative StoreID from helper.
1775              */
1776             debugs(88, 3, "storeId: " << http->storeEntry()->mem_obj->storeId());
1777         }
1778
1779         sc = storeClientListAdd(http->storeEntry(), this);
1780 #if USE_DELAY_POOLS
1781         sc->setDelayId(DelayId::DelayClient(http));
1782 #endif
1783
1784         assert(http->logType == LOG_TCP_HIT);
1785         reqofs = 0;
1786         /* guarantee nothing has been sent yet! */
1787         assert(http->out.size == 0);
1788         assert(http->out.offset == 0);
1789
1790         if (Ip::Qos::TheConfig.isHitTosActive()) {
1791             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1792         }
1793
1794         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1795             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1796         }
1797
1798         localTempBuffer.offset = reqofs;
1799         localTempBuffer.length = getNextNode()->readBuffer.length;
1800         localTempBuffer.data = getNextNode()->readBuffer.data;
1801         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1802     } else {
1803         /* MISS CASE, http->logType is already set! */
1804         processMiss();
1805     }
1806 }
1807
1808 /** The next node has removed itself from the stream. */
1809 void
1810 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1811 {
1812     /** detach from the stream */
1813     clientStreamDetach(node, http);
1814 }
1815
1816 /**
1817  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1818  * such, writes processed message to the message recipient
1819  */
1820 void
1821 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1822 {
1823     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1824     context->sendMoreData (result);
1825 }
1826
1827 void
1828 clientReplyContext::makeThisHead()
1829 {
1830     /* At least, I think thats what this does */
1831     dlinkDelete(&http->active, &ClientActiveRequests);
1832     dlinkAdd(http, &http->active, &ClientActiveRequests);
1833 }
1834
1835 bool
1836 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1837 {
1838     return /* aborted request */
1839         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1840         /* Upstream read error */ (result.flags.error) ||
1841         /* Upstream EOF */ (sizeToProcess == 0);
1842 }
1843
1844 void
1845 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1846 {
1847     /** call clientWriteComplete so the client socket gets closed
1848      *
1849      * We call into the stream, because we don't know that there is a
1850      * client socket!
1851      */
1852     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1853     StoreIOBuffer localTempBuffer;
1854     flags.complete = 1;
1855     http->request->flags.streamError = true;
1856     localTempBuffer.flags.error = result.flags.error;
1857     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1858                          localTempBuffer);
1859 }
1860
1861 void
1862 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1863 {
1864     StoreIOBuffer localTempBuffer;
1865
1866     if (result.length == 0) {
1867         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1868         flags.complete = 1;
1869     }
1870
1871     assert(result.offset - headers_sz == next()->readBuffer.offset);
1872     localTempBuffer.offset = result.offset - headers_sz;
1873     localTempBuffer.length = result.length;
1874
1875     if (localTempBuffer.length)
1876         localTempBuffer.data = source;
1877
1878     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1879                          localTempBuffer);
1880 }
1881
1882 clientStreamNode *
1883 clientReplyContext::next() const
1884 {
1885     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1886     return getNextNode();
1887 }
1888
1889 void
1890 clientReplyContext::sendBodyTooLargeError()
1891 {
1892     Ip::Address tmp_noaddr;
1893     tmp_noaddr.setNoAddr(); // TODO: make a global const
1894     http->logType = LOG_TCP_DENIED_REPLY;
1895     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1896                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1897                                        http->request);
1898     removeClientStoreReference(&(sc), http);
1899     HTTPMSGUNLOCK(reply);
1900     startError(err);
1901
1902 }
1903
1904 /// send 412 (Precondition Failed) to client
1905 void
1906 clientReplyContext::sendPreconditionFailedError()
1907 {
1908     http->logType = LOG_TCP_HIT;
1909     ErrorState *const err =
1910         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1911                          NULL, http->getConn()->clientConnection->remote, http->request);
1912     removeClientStoreReference(&sc, http);
1913     HTTPMSGUNLOCK(reply);
1914     startError(err);
1915 }
1916
1917 /// send 304 (Not Modified) to client
1918 void
1919 clientReplyContext::sendNotModified()
1920 {
1921     StoreEntry *e = http->storeEntry();
1922     const time_t timestamp = e->timestamp;
1923     HttpReply *const temprep = e->getReply()->make304();
1924     http->logType = LOG_TCP_IMS_HIT;
1925     removeClientStoreReference(&sc, http);
1926     createStoreEntry(http->request->method, RequestFlags());
1927     e = http->storeEntry();
1928     // Copy timestamp from the original entry so the 304
1929     // reply has a meaningful Age: header.
1930     e->timestampsSet();
1931     e->timestamp = timestamp;
1932     e->replaceHttpReply(temprep);
1933     e->complete();
1934     /*
1935      * TODO: why put this in the store and then serialise it and
1936      * then parse it again. Simply mark the request complete in
1937      * our context and write the reply struct to the client side.
1938      */
1939     triggerInitialStoreRead();
1940 }
1941
1942 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1943 /// depending on request method
1944 void
1945 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1946 {
1947     if (http->request->method == Http::METHOD_GET ||
1948             http->request->method == Http::METHOD_HEAD)
1949         sendNotModified();
1950     else
1951         sendPreconditionFailedError();
1952 }
1953
1954 void
1955 clientReplyContext::processReplyAccess ()
1956 {
1957     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
1958     assert(reply);
1959
1960     /** Don't block our own responses or HTTP status messages */
1961     if (http->logType == LOG_TCP_DENIED ||
1962             http->logType == LOG_TCP_DENIED_REPLY ||
1963             alwaysAllowResponse(reply->sline.status())) {
1964         headers_sz = reply->hdr_sz;
1965         processReplyAccessResult(ACCESS_ALLOWED);
1966         return;
1967     }
1968
1969     /** Check for reply to big error */
1970     if (reply->expectedBodyTooLarge(*http->request)) {
1971         sendBodyTooLargeError();
1972         return;
1973     }
1974
1975     headers_sz = reply->hdr_sz;
1976
1977     /** check for absent access controls (permit by default) */
1978     if (!Config.accessList.reply) {
1979         processReplyAccessResult(ACCESS_ALLOWED);
1980         return;
1981     }
1982
1983     /** Process http_reply_access lists */
1984     ACLFilledChecklist *replyChecklist =
1985         clientAclChecklistCreate(Config.accessList.reply, http);
1986     replyChecklist->reply = reply;
1987     HTTPMSGLOCK(replyChecklist->reply);
1988     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
1989 }
1990
1991 void
1992 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
1993 {
1994     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
1995     me->processReplyAccessResult(rv);
1996 }
1997
1998 void
1999 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2000 {
2001     debugs(88, 2, "The reply for " << http->request->method
2002            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2003            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2004
2005     if (accessAllowed != ACCESS_ALLOWED) {
2006         ErrorState *err;
2007         err_type page_id;
2008         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2009
2010         http->logType = LOG_TCP_DENIED_REPLY;
2011
2012         if (page_id == ERR_NONE)
2013             page_id = ERR_ACCESS_DENIED;
2014
2015         Ip::Address tmp_noaddr;
2016         tmp_noaddr.setNoAddr();
2017         err = clientBuildError(page_id, Http::scForbidden, NULL,
2018                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2019                                http->request);
2020
2021         removeClientStoreReference(&sc, http);
2022
2023         HTTPMSGUNLOCK(reply);
2024
2025         startError(err);
2026
2027         return;
2028     }
2029
2030     /* Ok, the reply is allowed, */
2031     http->loggingEntry(http->storeEntry());
2032
2033     ssize_t body_size = reqofs - reply->hdr_sz;
2034     if (body_size < 0) {
2035         reqofs = reply->hdr_sz;
2036         body_size = 0;
2037     }
2038
2039     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2040            (int) body_size << " bytes after " << reply->hdr_sz <<
2041            " bytes of headers");
2042
2043 #if USE_SQUID_ESI
2044
2045     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2046             !alwaysAllowResponse(reply->sline.status()) &&
2047             esiEnableProcessing(reply)) {
2048         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2049         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2050                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2051     }
2052
2053 #endif
2054
2055     if (http->request->method == Http::METHOD_HEAD) {
2056         /* do not forward body for HEAD replies */
2057         body_size = 0;
2058         http->flags.done_copying = true;
2059         flags.complete = 1;
2060     }
2061
2062     assert (!flags.headersSent);
2063     flags.headersSent = true;
2064
2065     StoreIOBuffer localTempBuffer;
2066     char *buf = next()->readBuffer.data;
2067     char *body_buf = buf + reply->hdr_sz;
2068
2069     //Server side may disable ranges under some circumstances.
2070
2071     if ((!http->request->range))
2072         next()->readBuffer.offset = 0;
2073
2074     body_buf -= next()->readBuffer.offset;
2075
2076     if (next()->readBuffer.offset != 0) {
2077         if (next()->readBuffer.offset > body_size) {
2078             /* Can't use any of the body we received. send nothing */
2079             localTempBuffer.length = 0;
2080             localTempBuffer.data = NULL;
2081         } else {
2082             localTempBuffer.length = body_size - next()->readBuffer.offset;
2083             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2084         }
2085     } else {
2086         localTempBuffer.length = body_size;
2087         localTempBuffer.data = body_buf;
2088     }
2089
2090     /* TODO??: move the data in the buffer back by the request header size */
2091     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2092                          http, reply, localTempBuffer);
2093
2094     return;
2095 }
2096
2097 void
2098 clientReplyContext::sendMoreData (StoreIOBuffer result)
2099 {
2100     if (deleting)
2101         return;
2102
2103     StoreEntry *entry = http->storeEntry();
2104
2105     ConnStateData * conn = http->getConn();
2106
2107     // too late, our conn is closing
2108     // TODO: should we also quit?
2109     if (conn == NULL) {
2110         debugs(33,3, "not sending more data to a closed connection" );
2111         return;
2112     }
2113     if (!conn->isOpen()) {
2114         debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2115         return;
2116     }
2117     if (conn->pinning.zeroReply) {
2118         debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2119         return;
2120     }
2121
2122     char *buf = next()->readBuffer.data;
2123
2124     if (buf != result.data) {
2125         /* we've got to copy some data */
2126         assert(result.length <= next()->readBuffer.length);
2127         memcpy(buf, result.data, result.length);
2128     }
2129
2130     if (reqofs==0 && !logTypeIsATcpHit(http->logType) && Comm::IsConnOpen(conn->clientConnection)) {
2131         if (Ip::Qos::TheConfig.isHitTosActive()) {
2132             Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2133         }
2134         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2135             Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2136         }
2137     }
2138
2139     /* We've got the final data to start pushing... */
2140     flags.storelogiccomplete = 1;
2141
2142     reqofs += result.length;
2143
2144     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2145
2146     assert(http->request != NULL);
2147
2148     /* ESI TODO: remove this assert once everything is stable */
2149     assert(http->client_stream.head->data
2150            && cbdataReferenceValid(http->client_stream.head->data));
2151
2152     makeThisHead();
2153
2154     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2155            reqofs << " bytes (" << result.length <<
2156            " new bytes)");
2157     debugs(88, 5, "clientReplyContext::sendMoreData:"
2158            << conn->clientConnection <<
2159            " '" << entry->url() << "'" <<
2160            " out.offset=" << http->out.offset);
2161
2162     /* update size of the request */
2163     reqsize = reqofs;
2164
2165     if (errorInStream(result, reqofs)) {
2166         sendStreamError(result);
2167         return;
2168     }
2169
2170     if (flags.headersSent) {
2171         pushStreamData (result, buf);
2172         return;
2173     }
2174
2175     cloneReply();
2176
2177     /* handle headers */
2178
2179     if (Config.onoff.log_mime_hdrs) {
2180         size_t k;
2181
2182         if ((k = headersEnd(buf, reqofs))) {
2183             safe_free(http->al->headers.reply);
2184             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2185             xstrncpy(http->al->headers.reply, buf, k);
2186         }
2187     }
2188
2189     holdingBuffer = result;
2190     processReplyAccess();
2191     return;
2192 }
2193
2194 /* Using this breaks the client layering just a little!
2195  */
2196 void
2197 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2198 {
2199     assert(http != NULL);
2200     /*
2201      * For erroneous requests, we might not have a h->request,
2202      * so make a fake one.
2203      */
2204
2205     if (http->request == NULL) {
2206         http->request = new HttpRequest(m, AnyP::PROTO_NONE, null_string);
2207         HTTPMSGLOCK(http->request);
2208     }
2209
2210     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2211
2212     // Make entry collapsable ASAP, to increase collapsing chances for others,
2213     // TODO: every must-revalidate and similar request MUST reach the origin,
2214     // but do we have to prohibit others from collapsing on that request?
2215     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2216             !reqFlags.needValidation &&
2217             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2218         // make the entry available for future requests now
2219         Store::Root().allowCollapsing(e, reqFlags, m);
2220     }
2221
2222     sc = storeClientListAdd(e, this);
2223
2224 #if USE_DELAY_POOLS
2225     sc->setDelayId(DelayId::DelayClient(http));
2226 #endif
2227
2228     reqofs = 0;
2229
2230     reqsize = 0;
2231
2232     /* I don't think this is actually needed! -- adrian */
2233     /* http->reqbuf = http->norm_reqbuf; */
2234     //    assert(http->reqbuf == http->norm_reqbuf);
2235     /* The next line is illegal because we don't know if the client stream
2236      * buffers have been set up
2237      */
2238     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2239     //        SendMoreData, this);
2240     /* So, we mark the store logic as complete */
2241     flags.storelogiccomplete = 1;
2242
2243     /* and get the caller to request a read, from whereever they are */
2244     /* NOTE: after ANY data flows down the pipe, even one step,
2245      * this function CAN NOT be used to manage errors
2246      */
2247     http->storeEntry(e);
2248 }
2249
2250 ErrorState *
2251 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2252                  Ip::Address &src_addr, HttpRequest * request)
2253 {
2254     ErrorState *err = new ErrorState(page_id, status, request);
2255     err->src_addr = src_addr;
2256
2257     if (url)
2258         err->url = xstrdup(url);
2259
2260     return err;
2261 }
2262