src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "http/Stream.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47 #if USE_SQUID_ESI
  48 #include "esi/Esi.h"
  49 #endif
  50
  51 #include <memory>
  52
  53 CBDATA_CLASS_INIT(clientReplyContext);
  54
  55 /* Local functions */
  56 extern "C" CSS clientReplyStatus;
  57 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  58
  59 /* privates */
  60
  61 clientReplyContext::~clientReplyContext()
  62 {
  63     deleting = true;
  64     /* This may trigger a callback back into SendMoreData as the cbdata
  65      * is still valid
  66      */
  67     removeClientStoreReference(&sc, http);
  68     /* old_entry might still be set if we didn't yet get the reply
  69      * code in HandleIMSReply() */
  70     removeStoreReference(&old_sc, &old_entry);
  71     safe_free(tempBuffer.data);
  72     cbdataReferenceDone(http);
  73     HTTPMSGUNLOCK(reply);
  74 }
  75
  76 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  77     purgeStatus(Http::scNone),
  78     lookingforstore(0),
  79     http(cbdataReference(clientContext)),
  80     headers_sz(0),
  81     sc(NULL),
  82     old_reqsize(0),
  83     reqsize(0),
  84     reqofs(0),
  85 #if USE_CACHE_DIGESTS
  86     lookup_type(NULL),
  87 #endif
  88     ourNode(NULL),
  89     reply(NULL),
  90     old_entry(NULL),
  91     old_sc(NULL),
  92     deleting(false),
  93     collapsedRevalidation(crNone)
  94 {
  95     *tempbuf = 0;
  96 }
  97
  98 /** Create an error in the store awaiting the client side to read it.
  99  *
 100  * This may be better placed in the clientStream logic, but it has not been
 101  * relocated there yet
 102  */
 103 void
 104 clientReplyContext::setReplyToError(
 105     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 106     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 107 #if USE_AUTH
 108     Auth::UserRequest::Pointer auth_user_request
 109 #else
 110     void*
 111 #endif
 112 )
 113 {
 114     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 115
 116     if (unparsedrequest)
 117         errstate->request_hdrs = xstrdup(unparsedrequest);
 118
 119 #if USE_AUTH
 120     errstate->auth_user_request = auth_user_request;
 121 #endif
 122     setReplyToError(method, errstate);
 123 }
 124
 125 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 126 {
 127     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 128         /* prevent confusion over whether we default to persistent or not */
 129         http->request->flags.proxyKeepalive = false;
 130
 131     http->al->http.code = errstate->httpStatus;
 132
 133     if (http->request)
 134         http->request->ignoreRange("responding with a Squid-generated error");
 135
 136     createStoreEntry(method, RequestFlags());
 137     assert(errstate->callback_data == NULL);
 138     errorAppendEntry(http->storeEntry(), errstate);
 139     /* Now the caller reads to get this */
 140 }
 141
 142 void
 143 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 144 {
 145     Must(futureReply);
 146     http->al->http.code = futureReply->sline.status();
 147
 148     HttpRequestMethod method;
 149     if (http->request) { // nil on responses to unparsable requests
 150         http->request->ignoreRange("responding with a Squid-generated reply");
 151         method = http->request->method;
 152     }
 153
 154     createStoreEntry(method, RequestFlags());
 155
 156     http->storeEntry()->storeErrorResponse(futureReply);
 157     /* Now the caller reads to get futureReply */
 158 }
 159
 160 // Assumes that the entry contains an error response without Content-Range.
 161 // To use with regular entries, make HTTP Range header removal conditional.
 162 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 163 {
 164     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 165     sc = storeClientListAdd(entry, this);
 166 #if USE_DELAY_POOLS
 167     sc->setDelayId(DelayId::DelayClient(http));
 168 #endif
 169     reqofs = 0;
 170     reqsize = 0;
 171     if (http->request)
 172         http->request->ignoreRange(reason);
 173     flags.storelogiccomplete = 1;
 174     http->storeEntry(entry);
 175 }
 176
 177 void
 178 clientReplyContext::removeStoreReference(store_client ** scp,
 179         StoreEntry ** ep)
 180 {
 181     StoreEntry *e;
 182     store_client *sc_tmp = *scp;
 183
 184     if ((e = *ep) != NULL) {
 185         *ep = NULL;
 186         storeUnregister(sc_tmp, e, this);
 187         *scp = NULL;
 188         e->unlock("clientReplyContext::removeStoreReference");
 189     }
 190 }
 191
 192 void
 193 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 194 {
 195     StoreEntry *reference = aHttpRequest->storeEntry();
 196     removeStoreReference(scp, &reference);
 197     aHttpRequest->storeEntry(reference);
 198 }
 199
 200 void
 201 clientReplyContext::saveState()
 202 {
 203     assert(old_sc == NULL);
 204     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 205     old_entry = http->storeEntry();
 206     old_sc = sc;
 207     old_reqsize = reqsize;
 208     tempBuffer.offset = reqofs;
 209     /* Prevent accessing the now saved entries */
 210     http->storeEntry(NULL);
 211     sc = NULL;
 212     reqsize = 0;
 213     reqofs = 0;
 214 }
 215
 216 void
 217 clientReplyContext::restoreState()
 218 {
 219     assert(old_sc != NULL);
 220     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 221     removeClientStoreReference(&sc, http);
 222     http->storeEntry(old_entry);
 223     sc = old_sc;
 224     reqsize = old_reqsize;
 225     reqofs = tempBuffer.offset;
 226     /* Prevent accessed the old saved entries */
 227     old_entry = NULL;
 228     old_sc = NULL;
 229     old_reqsize = 0;
 230     tempBuffer.offset = 0;
 231 }
 232
 233 void
 234 clientReplyContext::startError(ErrorState * err)
 235 {
 236     createStoreEntry(http->request->method, RequestFlags());
 237     triggerInitialStoreRead();
 238     errorAppendEntry(http->storeEntry(), err);
 239 }
 240
 241 clientStreamNode *
 242 clientReplyContext::getNextNode() const
 243 {
 244     return (clientStreamNode *)ourNode->node.next->data;
 245 }
 246
 247 /* This function is wrong - the client parameters don't include the
 248  * header offset
 249  */
 250 void
 251 clientReplyContext::triggerInitialStoreRead()
 252 {
 253     /* when confident, 0 becomes reqofs, and then this factors into
 254      * startSendProcess
 255      */
 256     assert(reqofs == 0);
 257     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 258     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 259 }
 260
 261 /* there is an expired entry in the store.
 262  * setup a temporary buffer area and perform an IMS to the origin
 263  */
 264 void
 265 clientReplyContext::processExpired()
 266 {
 267     const char *url = storeId();
 268     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 269     const time_t lastmod = http->storeEntry()->lastModified();
 270     assert(lastmod >= 0);
 271     /*
 272      * check if we are allowed to contact other servers
 273      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 274      *      a stale entry *if* it matches client requirements
 275      */
 276
 277     if (http->onlyIfCached()) {
 278         processOnlyIfCachedMiss();
 279         return;
 280     }
 281
 282     http->logType = LOG_TCP_REFRESH;
 283     http->request->flags.refresh = true;
 284 #if STORE_CLIENT_LIST_DEBUG
 285     /* Prevent a race with the store client memory free routines
 286      */
 287     assert(storeClientIsThisAClient(sc, this));
 288 #endif
 289     /* Prepare to make a new temporary request */
 290     saveState();
 291
 292     // TODO: support collapsed revalidation for Vary-controlled entries
 293     const bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
 294                                    !Store::Root().smpAware() &&
 295                                    http->request->vary_headers.isEmpty();
 296
 297     StoreEntry *entry = nullptr;
 298     if (collapsingAllowed) {
 299         if ((entry = storeGetPublicByRequest(http->request, ksRevalidation)))
 300             entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
 301     }
 302
 303     if (entry) {
 304         debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
 305         collapsedRevalidation = crSlave;
 306     } else {
 307         entry = storeCreateEntry(url,
 308                                  http->log_uri, http->request->flags, http->request->method);
 309         /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 310
 311         if (collapsingAllowed) {
 312             debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
 313             Store::Root().allowCollapsing(entry, http->request->flags,
 314                                           http->request->method);
 315             collapsedRevalidation = crInitiator;
 316         } else {
 317             collapsedRevalidation = crNone;
 318         }
 319     }
 320
 321     sc = storeClientListAdd(entry, this);
 322 #if USE_DELAY_POOLS
 323     /* delay_id is already set on original store client */
 324     sc->setDelayId(DelayId::DelayClient(http));
 325 #endif
 326
 327     http->request->lastmod = lastmod;
 328
 329     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 330         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 331         if (old_entry->hasEtag(etag) && !etag.weak)
 332             http->request->etag = etag.str;
 333     }
 334
 335     debugs(88, 5, "lastmod " << entry->lastModified());
 336     http->storeEntry(entry);
 337     assert(http->out.offset == 0);
 338     assert(http->request->clientConnectionManager == http->getConn());
 339
 340     if (collapsedRevalidation != crSlave) {
 341         /*
 342          * A refcounted pointer so that FwdState stays around as long as
 343          * this clientReplyContext does
 344          */
 345         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 346         FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 347     }
 348     /* Register with storage manager to receive updates when data comes in. */
 349
 350     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 351         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 352
 353     {
 354         /* start counting the length from 0 */
 355         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 356         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 357     }
 358 }
 359
 360 void
 361 clientReplyContext::sendClientUpstreamResponse()
 362 {
 363     StoreIOBuffer tempresult;
 364     removeStoreReference(&old_sc, &old_entry);
 365     /* here the data to send is the data we just received */
 366     tempBuffer.offset = 0;
 367     old_reqsize = 0;
 368     /* sendMoreData tracks the offset as well.
 369      * Force it back to zero */
 370     reqofs = 0;
 371     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 372     /* TODO: provide sendMoreData with the ready parsed reply */
 373     tempresult.length = reqsize;
 374     tempresult.data = tempbuf;
 375     sendMoreData(tempresult);
 376 }
 377
 378 void
 379 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 380 {
 381     clientReplyContext *context = (clientReplyContext *)data;
 382     context->handleIMSReply(result);
 383 }
 384
 385 void
 386 clientReplyContext::sendClientOldEntry()
 387 {
 388     /* Get the old request back */
 389     restoreState();
 390     /* here the data to send is in the next nodes buffers already */
 391     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 392     /* sendMoreData tracks the offset as well.
 393      * Force it back to zero */
 394     reqofs = 0;
 395     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 396     sendMoreData(tempresult);
 397 }
 398
 399 /* This is the workhorse of the HandleIMSReply callback.
 400  *
 401  * It is called when we've got data back from the origin following our
 402  * IMS request to revalidate a stale entry.
 403  */
 404 void
 405 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 406 {
 407     if (deleting)
 408         return;
 409
 410     debugs(88, 3, http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes");
 411
 412     if (http->storeEntry() == NULL)
 413         return;
 414
 415     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 416         return;
 417
 418     if (collapsedRevalidation == crSlave && EBIT_TEST(http->storeEntry()->flags, KEY_PRIVATE)) {
 419         debugs(88, 3, "CF slave hit private " << *http->storeEntry() << ". MISS");
 420         // restore context to meet processMiss() expectations
 421         restoreState();
 422         http->logType = LOG_TCP_MISS;
 423         processMiss();
 424         return;
 425     }
 426
 427     /* update size of the request */
 428     reqsize = result.length + reqofs;
 429
 430     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 431
 432     // request to origin was aborted
 433     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 434         debugs(88, 3, "request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client");
 435         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 436         sendClientOldEntry();
 437     }
 438
 439     const HttpReply *old_rep = old_entry->getReply();
 440
 441     // origin replied 304
 442     if (status == Http::scNotModified) {
 443         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 444         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 445
 446         // update headers on existing entry
 447         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 448
 449         // if client sent IMS
 450
 451         if (http->request->flags.ims && !old_entry->modifiedSince(http->request->ims, http->request->imslen)) {
 452             // forward the 304 from origin
 453             debugs(88, 3, "origin replied 304, revalidating existing entry and forwarding 304 to client");
 454             sendClientUpstreamResponse();
 455         } else {
 456             // send existing entry, it's still valid
 457             debugs(88, 3, "origin replied 304, revalidating existing entry and sending " <<
 458                    old_rep->sline.status() << " to client");
 459             sendClientOldEntry();
 460         }
 461     }
 462
 463     // origin replied with a non-error code
 464     else if (status > Http::scNone && status < Http::scInternalServerError) {
 465         const HttpReply *new_rep = http->storeEntry()->getReply();
 466         // RFC 7234 section 4: a cache MUST use the most recent response
 467         // (as determined by the Date header field)
 468         if (new_rep->olderThan(old_rep)) {
 469             http->logType.err.ignored = true;
 470             debugs(88, 3, "origin replied " << status <<
 471                    " but with an older date header, sending old entry (" <<
 472                    old_rep->sline.status() << ") to client");
 473             sendClientOldEntry();
 474         } else {
 475             http->logType = LOG_TCP_REFRESH_MODIFIED;
 476             debugs(88, 3, "origin replied " << status <<
 477                    ", replacing existing entry and forwarding to client");
 478
 479             if (collapsedRevalidation)
 480                 http->storeEntry()->clearPublicKeyScope();
 481
 482             sendClientUpstreamResponse();
 483         }
 484     }
 485
 486     // origin replied with an error
 487     else if (http->request->flags.failOnValidationError) {
 488         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 489         debugs(88, 3, "origin replied with error " << status <<
 490                ", forwarding to client due to fail_on_validation_err");
 491         sendClientUpstreamResponse();
 492     } else {
 493         // ignore and let client have old entry
 494         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 495         debugs(88, 3, "origin replied with error " <<
 496                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 497         sendClientOldEntry();
 498     }
 499 }
 500
 501 SQUIDCEXTERN CSR clientGetMoreData;
 502 SQUIDCEXTERN CSD clientReplyDetach;
 503
 504 /**
 505  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 506  * have been parsed.  Normally this should be a single call, but
 507  * it might take more than one.  As soon as we have the headers,
 508  * we hand off to clientSendMoreData, processExpired, or
 509  * processMiss.
 510  */
 511 void
 512 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 513 {
 514     clientReplyContext *context = (clientReplyContext *)data;
 515     context->cacheHit(result);
 516 }
 517
 518 /**
 519  * Process a possible cache HIT.
 520  */
 521 void
 522 clientReplyContext::cacheHit(StoreIOBuffer result)
 523 {
 524     /** Ignore if the HIT object is being deleted. */
 525     if (deleting) {
 526         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 527         return;
 528     }
 529
 530     StoreEntry *e = http->storeEntry();
 531
 532     HttpRequest *r = http->request;
 533
 534     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 535
 536     if (http->storeEntry() == NULL) {
 537         debugs(88, 3, "clientCacheHit: request aborted");
 538         return;
 539     } else if (result.flags.error) {
 540         /* swap in failure */
 541         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 542         http->logType = LOG_TCP_SWAPFAIL_MISS;
 543         removeClientStoreReference(&sc, http);
 544         processMiss();
 545         return;
 546     }
 547
 548     // The previously identified hit suddenly became unsharable!
 549     // This is common for collapsed forwarding slaves but might also
 550     // happen to regular hits because we are called asynchronously.
 551     if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
 552         debugs(88, 3, "unsharable " << *e << ". MISS");
 553         http->logType = LOG_TCP_MISS;
 554         processMiss();
 555         return;
 556     }
 557
 558     if (result.length == 0) {
 559         debugs(88, 5, "store IO buffer has no content. MISS");
 560         /* the store couldn't get enough data from the file for us to id the
 561          * object
 562          */
 563         /* treat as a miss */
 564         http->logType = LOG_TCP_MISS;
 565         processMiss();
 566         return;
 567     }
 568
 569     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 570     /* update size of the request */
 571     reqsize = result.length + reqofs;
 572
 573     /*
 574      * Got the headers, now grok them
 575      */
 576     assert(http->logType.oldType == LOG_TCP_HIT);
 577
 578     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 579         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 580         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 581         processMiss();
 582         return;
 583     }
 584
 585     switch (varyEvaluateMatch(e, r)) {
 586
 587     case VARY_NONE:
 588         /* No variance detected. Continue as normal */
 589         break;
 590
 591     case VARY_MATCH:
 592         /* This is the correct entity for this request. Continue */
 593         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 594         break;
 595
 596     case VARY_OTHER:
 597         /* This is not the correct entity for this request. We need
 598          * to requery the cache.
 599          */
 600         removeClientStoreReference(&sc, http);
 601         e = NULL;
 602         /* Note: varyEvalyateMatch updates the request with vary information
 603          * so we only get here once. (it also takes care of cancelling loops)
 604          */
 605         debugs(88, 2, "clientProcessHit: Vary detected!");
 606         clientGetMoreData(ourNode, http);
 607         return;
 608
 609     case VARY_CANCEL:
 610         /* varyEvaluateMatch found a object loop. Process as miss */
 611         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 612         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 613         processMiss();
 614         return;
 615     }
 616
 617     if (r->method == Http::METHOD_PURGE) {
 618         debugs(88, 5, "PURGE gets a HIT");
 619         removeClientStoreReference(&sc, http);
 620         e = NULL;
 621         purgeRequest();
 622         return;
 623     }
 624
 625     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 626         debugs(88, 5, "negative-HIT");
 627         http->logType = LOG_TCP_NEGATIVE_HIT;
 628         sendMoreData(result);
 629         return;
 630     } else if (blockedHit()) {
 631         debugs(88, 5, "send_hit forces a MISS");
 632         http->logType = LOG_TCP_MISS;
 633         processMiss();
 634         return;
 635     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 636         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 637         /*
 638          * We hold a stale copy; it needs to be validated
 639          */
 640         /*
 641          * The 'needValidation' flag is used to prevent forwarding
 642          * loops between siblings.  If our copy of the object is stale,
 643          * then we should probably only use parents for the validation
 644          * request.  Otherwise two siblings could generate a loop if
 645          * both have a stale version of the object.
 646          */
 647         r->flags.needValidation = true;
 648
 649         if (e->lastModified() < 0) {
 650             debugs(88, 3, "validate HIT object? NO. Can't calculate entry modification time. Do MISS.");
 651             /*
 652              * We cannot revalidate entries without knowing their
 653              * modification time.
 654              * XXX: BUG 1890 objects without Date do not get one added.
 655              */
 656             http->logType = LOG_TCP_MISS;
 657             processMiss();
 658         } else if (r->flags.noCache) {
 659             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 660             /*
 661              * This did not match a refresh pattern that overrides no-cache
 662              * we should honour the client no-cache header.
 663              */
 664             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 665             processMiss();
 666         } else if (r->url.getScheme() == AnyP::PROTO_HTTP) {
 667             debugs(88, 3, "validate HIT object? YES.");
 668             /*
 669              * Object needs to be revalidated
 670              * XXX This could apply to FTP as well, if Last-Modified is known.
 671              */
 672             processExpired();
 673         } else {
 674             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 675             /*
 676              * We don't know how to re-validate other protocols. Handle
 677              * them as if the object has expired.
 678              */
 679             http->logType = LOG_TCP_MISS;
 680             processMiss();
 681         }
 682         return;
 683     } else if (r->conditional()) {
 684         debugs(88, 5, "conditional HIT");
 685         if (processConditional(result))
 686             return;
 687     }
 688
 689     /*
 690      * plain ol' cache hit
 691      */
 692     debugs(88, 5, "plain old HIT");
 693
 694 #if USE_DELAY_POOLS
 695     if (e->store_status != STORE_OK)
 696         http->logType = LOG_TCP_MISS;
 697     else
 698 #endif
 699         if (e->mem_status == IN_MEMORY)
 700             http->logType = LOG_TCP_MEM_HIT;
 701         else if (Config.onoff.offline)
 702             http->logType = LOG_TCP_OFFLINE_HIT;
 703
 704     sendMoreData(result);
 705 }
 706
 707 /**
 708  * Prepare to fetch the object as it's a cache miss of some kind.
 709  */
 710 void
 711 clientReplyContext::processMiss()
 712 {
 713     char *url = http->uri;
 714     HttpRequest *r = http->request;
 715     ErrorState *err = NULL;
 716     debugs(88, 4, r->method << ' ' << url);
 717
 718     /**
 719      * We might have a left-over StoreEntry from a failed cache hit
 720      * or IMS request.
 721      */
 722     if (http->storeEntry()) {
 723         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 724             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 725             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 726             http->storeEntry()->dump(1);
 727         }
 728
 729         removeClientStoreReference(&sc, http);
 730     }
 731
 732     /** Check if its a PURGE request to be actioned. */
 733     if (r->method == Http::METHOD_PURGE) {
 734         purgeRequest();
 735         return;
 736     }
 737
 738     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 739     if (r->method == Http::METHOD_OTHER) {
 740         purgeAllCached();
 741     }
 742
 743     /** Check if 'only-if-cached' flag is set. Action if so. */
 744     if (http->onlyIfCached()) {
 745         processOnlyIfCachedMiss();
 746         return;
 747     }
 748
 749     /// Deny loops
 750     if (r->flags.loopDetected) {
 751         http->al->http.code = Http::scForbidden;
 752         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 753         createStoreEntry(r->method, RequestFlags());
 754         errorAppendEntry(http->storeEntry(), err);
 755         triggerInitialStoreRead();
 756         return;
 757     } else {
 758         assert(http->out.offset == 0);
 759         createStoreEntry(r->method, r->flags);
 760         triggerInitialStoreRead();
 761
 762         if (http->redirect.status) {
 763             HttpReply *rep = new HttpReply;
 764             http->logType = LOG_TCP_REDIRECT;
 765             http->storeEntry()->releaseRequest();
 766             rep->redirect(http->redirect.status, http->redirect.location);
 767             http->storeEntry()->replaceHttpReply(rep);
 768             http->storeEntry()->complete();
 769             return;
 770         }
 771
 772         assert(r->clientConnectionManager == http->getConn());
 773
 774         /** Start forwarding to get the new object from network */
 775         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 776         FwdState::Start(conn, http->storeEntry(), r, http->al);
 777     }
 778 }
 779
 780 /**
 781  * client issued a request with an only-if-cached cache-control directive;
 782  * we did not find a cached object that can be returned without
 783  *     contacting other servers;
 784  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 785  */
 786 void
 787 clientReplyContext::processOnlyIfCachedMiss()
 788 {
 789     debugs(88, 4, http->request->method << ' ' << http->uri);
 790     http->al->http.code = Http::scGatewayTimeout;
 791     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 792                                        http->getConn()->clientConnection->remote, http->request);
 793     removeClientStoreReference(&sc, http);
 794     startError(err);
 795 }
 796
 797 /// process conditional request from client
 798 bool
 799 clientReplyContext::processConditional(StoreIOBuffer &result)
 800 {
 801     StoreEntry *const e = http->storeEntry();
 802
 803     if (e->getReply()->sline.status() != Http::scOkay) {
 804         debugs(88, 4, "Reply code " << e->getReply()->sline.status() << " != 200");
 805         http->logType = LOG_TCP_MISS;
 806         processMiss();
 807         return true;
 808     }
 809
 810     HttpRequest &r = *http->request;
 811
 812     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 813         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 814         sendPreconditionFailedError();
 815         return true;
 816     }
 817
 818     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 819         // RFC 7232: If-None-Match recipient MUST ignore IMS
 820         r.flags.ims = false;
 821         r.ims = -1;
 822         r.imslen = 0;
 823         r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 824
 825         if (e->hasIfNoneMatchEtag(r)) {
 826             sendNotModifiedOrPreconditionFailedError();
 827             return true;
 828         }
 829
 830         // None-Match is true (no ETag matched); treat as an unconditional hit
 831         return false;
 832     }
 833
 834     if (r.flags.ims) {
 835         // handle If-Modified-Since requests from the client
 836         if (e->modifiedSince(r.ims, r.imslen)) {
 837             // Modified-Since is true; treat as an unconditional hit
 838             return false;
 839
 840         } else {
 841             // otherwise reply with 304 Not Modified
 842             sendNotModified();
 843         }
 844         return true;
 845     }
 846
 847     return false;
 848 }
 849
 850 /// whether squid.conf send_hit prevents us from serving this hit
 851 bool
 852 clientReplyContext::blockedHit() const
 853 {
 854     if (!Config.accessList.sendHit)
 855         return false; // hits are not blocked by default
 856
 857     if (http->flags.internal)
 858         return false; // internal content "hits" cannot be blocked
 859
 860     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 861         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 862         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 863         HTTPMSGLOCK(chl->reply);
 864         return chl->fastCheck() != ACCESS_ALLOWED; // when in doubt, block
 865     }
 866
 867     // This does not happen, I hope, because we are called from CacheHit, which
 868     // is called via a storeClientCopy() callback, and store should initialize
 869     // the reply before calling that callback.
 870     debugs(88, 3, "Missing reply!");
 871     return false;
 872 }
 873
 874 void
 875 clientReplyContext::purgeRequestFindObjectToPurge()
 876 {
 877     /* Try to find a base entry */
 878     http->flags.purging = true;
 879     lookingforstore = 1;
 880
 881     // TODO: can we use purgeAllCached() here instead of doing the
 882     // getPublicByRequestMethod() dance?
 883     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 884 }
 885
 886 // Purges all entries with a given url
 887 // TODO: move to SideAgent parent, when we have one
 888 /*
 889  * We probably cannot purge Vary-affected responses because their MD5
 890  * keys depend on vary headers.
 891  */
 892 void
 893 purgeEntriesByUrl(HttpRequest * req, const char *url)
 894 {
 895 #if USE_HTCP
 896     bool get_or_head_sent = false;
 897 #endif
 898
 899     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 900         if (m.respMaybeCacheable()) {
 901             if (StoreEntry *entry = storeGetPublic(url, m)) {
 902                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 903 #if USE_HTCP
 904                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 905                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 906                     get_or_head_sent = true;
 907                 }
 908 #endif
 909                 entry->release();
 910             }
 911         }
 912     }
 913
 914 #if USE_HTCP
 915     if (!get_or_head_sent) {
 916         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 917     }
 918 #endif
 919 }
 920
 921 void
 922 clientReplyContext::purgeAllCached()
 923 {
 924     // XXX: performance regression, c_str() reallocates
 925     SBuf url(http->request->effectiveRequestUri());
 926     purgeEntriesByUrl(http->request, url.c_str());
 927 }
 928
 929 void
 930 clientReplyContext::created(StoreEntry *newEntry)
 931 {
 932     if (lookingforstore == 1)
 933         purgeFoundGet(newEntry);
 934     else if (lookingforstore == 2)
 935         purgeFoundHead(newEntry);
 936     else if (lookingforstore == 3)
 937         purgeDoPurgeGet(newEntry);
 938     else if (lookingforstore == 4)
 939         purgeDoPurgeHead(newEntry);
 940     else if (lookingforstore == 5)
 941         identifyFoundObject(newEntry);
 942 }
 943
 944 void
 945 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 946 {
 947     if (newEntry->isNull()) {
 948         lookingforstore = 2;
 949         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 950     } else
 951         purgeFoundObject (newEntry);
 952 }
 953
 954 void
 955 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 956 {
 957     if (newEntry->isNull())
 958         purgeDoMissPurge();
 959     else
 960         purgeFoundObject (newEntry);
 961 }
 962
 963 void
 964 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 965 {
 966     assert (entry && !entry->isNull());
 967
 968     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 969         http->logType = LOG_TCP_DENIED;
 970         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 971                                            http->getConn()->clientConnection->remote, http->request);
 972         startError(err);
 973         return; // XXX: leaking unused entry if some store does not keep it
 974     }
 975
 976     StoreIOBuffer localTempBuffer;
 977     /* Swap in the metadata */
 978     http->storeEntry(entry);
 979
 980     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 981     http->storeEntry()->createMemObject(storeId(), http->log_uri,
 982                                         http->request->method);
 983
 984     sc = storeClientListAdd(http->storeEntry(), this);
 985
 986     http->logType = LOG_TCP_HIT;
 987
 988     reqofs = 0;
 989
 990     localTempBuffer.offset = http->out.offset;
 991
 992     localTempBuffer.length = next()->readBuffer.length;
 993
 994     localTempBuffer.data = next()->readBuffer.data;
 995
 996     storeClientCopy(sc, http->storeEntry(),
 997                     localTempBuffer, CacheHit, this);
 998 }
 999
1000 void
1001 clientReplyContext::purgeRequest()
1002 {
1003     debugs(88, 3, "Config2.onoff.enable_purge = " <<
1004            Config2.onoff.enable_purge);
1005
1006     if (!Config2.onoff.enable_purge) {
1007         http->logType = LOG_TCP_DENIED;
1008         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
1009         startError(err);
1010         return;
1011     }
1012
1013     /* Release both IP cache */
1014     ipcacheInvalidate(http->request->url.host());
1015
1016     if (!http->flags.purging)
1017         purgeRequestFindObjectToPurge();
1018     else
1019         purgeDoMissPurge();
1020 }
1021
1022 void
1023 clientReplyContext::purgeDoMissPurge()
1024 {
1025     http->logType = LOG_TCP_MISS;
1026     lookingforstore = 3;
1027     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
1028 }
1029
1030 void
1031 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
1032 {
1033     assert (newEntry);
1034     /* Move to new() when that is created */
1035     purgeStatus = Http::scNotFound;
1036
1037     if (!newEntry->isNull()) {
1038         /* Release the cached URI */
1039         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
1040 #if USE_HTCP
1041         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1042 #endif
1043         newEntry->release();
1044         purgeStatus = Http::scOkay;
1045     }
1046
1047     lookingforstore = 4;
1048     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
1049 }
1050
1051 void
1052 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
1053 {
1054     if (newEntry && !newEntry->isNull()) {
1055         debugs(88, 4, "HEAD " << newEntry->url());
1056 #if USE_HTCP
1057         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1058 #endif
1059         newEntry->release();
1060         purgeStatus = Http::scOkay;
1061     }
1062
1063     /* And for Vary, release the base URI if none of the headers was included in the request */
1064     if (!http->request->vary_headers.isEmpty()
1065             && http->request->vary_headers.find('=') != SBuf::npos) {
1066         // XXX: performance regression, c_str() reallocates
1067         SBuf tmp(http->request->effectiveRequestUri());
1068         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1069
1070         if (entry) {
1071             debugs(88, 4, "Vary GET " << entry->url());
1072 #if USE_HTCP
1073             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1074 #endif
1075             entry->release();
1076             purgeStatus = Http::scOkay;
1077         }
1078
1079         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1080
1081         if (entry) {
1082             debugs(88, 4, "Vary HEAD " << entry->url());
1083 #if USE_HTCP
1084             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1085 #endif
1086             entry->release();
1087             purgeStatus = Http::scOkay;
1088         }
1089     }
1090
1091     /*
1092      * Make a new entry to hold the reply to be written
1093      * to the client.
1094      */
1095     /* FIXME: This doesn't need to go through the store. Simply
1096      * push down the client chain
1097      */
1098     createStoreEntry(http->request->method, RequestFlags());
1099
1100     triggerInitialStoreRead();
1101
1102     HttpReply *rep = new HttpReply;
1103     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1104     http->storeEntry()->replaceHttpReply(rep);
1105     http->storeEntry()->complete();
1106 }
1107
1108 void
1109 clientReplyContext::traceReply(clientStreamNode * node)
1110 {
1111     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1112     StoreIOBuffer localTempBuffer;
1113     createStoreEntry(http->request->method, RequestFlags());
1114     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1115     localTempBuffer.length = nextNode->readBuffer.length;
1116     localTempBuffer.data = nextNode->readBuffer.data;
1117     storeClientCopy(sc, http->storeEntry(),
1118                     localTempBuffer, SendMoreData, this);
1119     http->storeEntry()->releaseRequest();
1120     http->storeEntry()->buffer();
1121     HttpReply *rep = new HttpReply;
1122     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1123     http->storeEntry()->replaceHttpReply(rep);
1124     http->request->swapOut(http->storeEntry());
1125     http->storeEntry()->complete();
1126 }
1127
1128 #define SENDING_BODY 0
1129 #define SENDING_HDRSONLY 1
1130 int
1131 clientReplyContext::checkTransferDone()
1132 {
1133     StoreEntry *entry = http->storeEntry();
1134
1135     if (entry == NULL)
1136         return 0;
1137
1138     /*
1139      * For now, 'done_copying' is used for special cases like
1140      * Range and HEAD requests.
1141      */
1142     if (http->flags.done_copying)
1143         return 1;
1144
1145     if (http->request->flags.chunkedReply && !flags.complete) {
1146         // last-chunk was not sent
1147         return 0;
1148     }
1149
1150     /*
1151      * Handle STORE_OK objects.
1152      * objectLen(entry) will be set proprely.
1153      * RC: Does objectLen(entry) include the Headers?
1154      * RC: Yes.
1155      */
1156     if (entry->store_status == STORE_OK) {
1157         return storeOKTransferDone();
1158     } else {
1159         return storeNotOKTransferDone();
1160     }
1161 }
1162
1163 int
1164 clientReplyContext::storeOKTransferDone() const
1165 {
1166     assert(http->storeEntry()->objectLen() >= 0);
1167     assert(http->storeEntry()->objectLen() >= headers_sz);
1168     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1169         debugs(88,3,HERE << "storeOKTransferDone " <<
1170                " out.offset=" << http->out.offset <<
1171                " objectLen()=" << http->storeEntry()->objectLen() <<
1172                " headers_sz=" << headers_sz);
1173         return 1;
1174     }
1175
1176     return 0;
1177 }
1178
1179 int
1180 clientReplyContext::storeNotOKTransferDone() const
1181 {
1182     /*
1183      * Now, handle STORE_PENDING objects
1184      */
1185     MemObject *mem = http->storeEntry()->mem_obj;
1186     assert(mem != NULL);
1187     assert(http->request != NULL);
1188     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1189     HttpReply const *curReply = mem->getReply();
1190
1191     if (headers_sz == 0)
1192         /* haven't found end of headers yet */
1193         return 0;
1194
1195     /*
1196      * Figure out how much data we are supposed to send.
1197      * If we are sending a body and we don't have a content-length,
1198      * then we must wait for the object to become STORE_OK.
1199      */
1200     if (curReply->content_length < 0)
1201         return 0;
1202
1203     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1204
1205     if (http->out.size < expectedLength)
1206         return 0;
1207     else {
1208         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1209                " out.size=" << http->out.size <<
1210                " expectedLength=" << expectedLength);
1211         return 1;
1212     }
1213 }
1214
1215 /* A write has completed, what is the next status based on the
1216  * canonical request data?
1217  * 1 something is wrong
1218  * 0 nothing is wrong.
1219  *
1220  */
1221 int
1222 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1223 {
1224 #if SIZEOF_INT64_T == 4
1225     if (http->out.size > 0x7FFF0000) {
1226         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1227         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1228         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1229         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1230         return 1;
1231     }
1232
1233     if (http->out.offset > 0x7FFF0000) {
1234         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1235         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1236         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1237         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1238         return 1;
1239     }
1240
1241 #endif
1242     return 0;
1243 }
1244
1245 /* Preconditions:
1246  * *http is a valid structure.
1247  * fd is either -1, or an open fd.
1248  *
1249  * TODO: enumify this
1250  *
1251  * This function is used by any http request sink, to determine the status
1252  * of the object.
1253  */
1254 clientStream_status_t
1255 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1256 {
1257     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1258     assert (context);
1259     assert (context->http == http);
1260     return context->replyStatus();
1261 }
1262
1263 clientStream_status_t
1264 clientReplyContext::replyStatus()
1265 {
1266     int done;
1267     /* Here because lower nodes don't need it */
1268
1269     if (http->storeEntry() == NULL) {
1270         debugs(88, 5, "clientReplyStatus: no storeEntry");
1271         return STREAM_FAILED;   /* yuck, but what can we do? */
1272     }
1273
1274     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1275         /* TODO: Could upstream read errors (result.flags.error) be
1276          * lost, and result in undersize requests being considered
1277          * complete. Should we tcp reset such connections ?
1278          */
1279         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1280         return STREAM_FAILED;
1281     }
1282
1283     if ((done = checkTransferDone()) != 0 || flags.complete) {
1284         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1285         /* Ok we're finished, but how? */
1286
1287         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1288             debugs(88, 5, "clientReplyStatus: truncated response body");
1289             return STREAM_UNPLANNED_COMPLETE;
1290         }
1291
1292         if (!done) {
1293             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1294             return STREAM_FAILED;
1295         }
1296
1297         const int64_t expectedBodySize =
1298             http->storeEntry()->getReply()->bodySize(http->request->method);
1299         if (expectedBodySize >= 0 && !http->gotEnough()) {
1300             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1301             return STREAM_UNPLANNED_COMPLETE;
1302         }
1303
1304         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1305                http->request->flags.proxyKeepalive);
1306         return STREAM_COMPLETE;
1307     }
1308
1309     // XXX: Should this be checked earlier? We could return above w/o checking.
1310     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1311         /* 4096 is a margin for the HTTP headers included in out.offset */
1312         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1313         return STREAM_FAILED;
1314     }
1315
1316     return STREAM_NONE;
1317 }
1318
1319 /* Responses with no body will not have a content-type header,
1320  * which breaks the rep_mime_type acl, which
1321  * coincidentally, is the most common acl for reply access lists.
1322  * A better long term fix for this is to allow acl matchs on the various
1323  * status codes, and then supply a default ruleset that puts these
1324  * codes before any user defines access entries. That way the user
1325  * can choose to block these responses where appropriate, but won't get
1326  * mysterious breakages.
1327  */
1328 bool
1329 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1330 {
1331     bool result;
1332
1333     switch (sline) {
1334
1335     case Http::scContinue:
1336
1337     case Http::scSwitchingProtocols:
1338
1339     case Http::scProcessing:
1340
1341     case Http::scNoContent:
1342
1343     case Http::scNotModified:
1344         result = true;
1345         break;
1346
1347     default:
1348         result = false;
1349     }
1350
1351     return result;
1352 }
1353
1354 /**
1355  * Generate the reply headers sent to client.
1356  *
1357  * Filters out unwanted entries and hop-by-hop from original reply header
1358  * then adds extra entries if we have more info than origin server
1359  * then adds Squid specific entries
1360  */
1361 void
1362 clientReplyContext::buildReplyHeader()
1363 {
1364     HttpHeader *hdr = &reply->header;
1365     const bool is_hit = http->logType.isTcpHit();
1366     HttpRequest *request = http->request;
1367 #if DONT_FILTER_THESE
1368     /* but you might want to if you run Squid as an HTTP accelerator */
1369     /* hdr->delById(HDR_ACCEPT_RANGES); */
1370     hdr->delById(HDR_ETAG);
1371 #endif
1372
1373     if (is_hit || collapsedRevalidation == crSlave)
1374         hdr->delById(Http::HdrType::SET_COOKIE);
1375     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1376
1377     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1378     // remove the Proxy-Authenticate header
1379     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0)) {
1380 #if USE_ADAPTATION
1381         // but allow adaptation services to authenticate clients
1382         // via request satisfaction
1383         if (!http->requestSatisfactionMode())
1384 #endif
1385             reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1386     }
1387
1388     reply->header.removeHopByHopEntries();
1389
1390     //    if (request->range)
1391     //      clientBuildRangeHeader(http, reply);
1392
1393     /*
1394      * Add a estimated Age header on cache hits.
1395      */
1396     if (is_hit) {
1397         /*
1398          * Remove any existing Age header sent by upstream caches
1399          * (note that the existing header is passed along unmodified
1400          * on cache misses)
1401          */
1402         hdr->delById(Http::HdrType::AGE);
1403         /*
1404          * This adds the calculated object age. Note that the details of the
1405          * age calculation is performed by adjusting the timestamp in
1406          * StoreEntry::timestampsSet(), not here.
1407          */
1408         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1409             hdr->delById(Http::HdrType::DATE);
1410             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1411         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1412             // Swap the Date: header to current time if we are simulating an origin
1413             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1414             if (h)
1415                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1416             hdr->delById(Http::HdrType::DATE);
1417             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1418             h = hdr->findEntry(Http::HdrType::EXPIRES);
1419             if (h && http->storeEntry()->expires >= 0) {
1420                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1421                 hdr->delById(Http::HdrType::EXPIRES);
1422                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1423             }
1424             if (http->storeEntry()->timestamp <= squid_curtime) {
1425                 // put X-Cache-Age: instead of Age:
1426                 char age[64];
1427                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1428                 hdr->putExt("X-Cache-Age", age);
1429             }
1430         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1431             hdr->putInt(Http::HdrType::AGE,
1432                         squid_curtime - http->storeEntry()->timestamp);
1433             /* Signal old objects.  NB: rfc 2616 is not clear,
1434              * by implication, on whether we should do this to all
1435              * responses, or only cache hits.
1436              * 14.46 states it ONLY applys for heuristically caclulated
1437              * freshness values, 13.2.4 doesn't specify the same limitation.
1438              * We interpret RFC 2616 under the combination.
1439              */
1440             /* TODO: if maxage or s-maxage is present, don't do this */
1441
1442             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1443                 char tbuf[512];
1444                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1445                           "113", ThisCache,
1446                           "This cache hit is still fresh and more than 1 day old");
1447                 hdr->putStr(Http::HdrType::WARNING, tbuf);
1448             }
1449         }
1450     }
1451
1452     /* RFC 2616: Section 14.18
1453      *
1454      * Add a Date: header if missing.
1455      * We have access to a clock therefore are required to amend any shortcoming in servers.
1456      *
1457      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1458      */
1459     if ( !hdr->has(Http::HdrType::DATE) ) {
1460         if (!http->storeEntry())
1461             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1462         else if (http->storeEntry()->timestamp > 0)
1463             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1464         else {
1465             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1466             /* dump something useful about the problem */
1467             http->storeEntry()->dump(DBG_IMPORTANT);
1468         }
1469     }
1470
1471     // add Warnings required by RFC 2616 if serving a stale hit
1472     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1473         hdr->putWarning(110, "Response is stale");
1474         if (http->request->flags.needValidation)
1475             hdr->putWarning(111, "Revalidation failed");
1476     }
1477
1478     /* Filter unproxyable authentication types */
1479     if (http->logType.oldType != LOG_TCP_DENIED &&
1480             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1481         HttpHeaderPos pos = HttpHeaderInitPos;
1482         HttpHeaderEntry *e;
1483
1484         int connection_auth_blocked = 0;
1485         while ((e = hdr->getEntry(&pos))) {
1486             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1487                 const char *value = e->value.rawBuf();
1488
1489                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1490                         (value[4] == '\0' || value[4] == ' '))
1491                         ||
1492                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1493                          (value[9] == '\0' || value[9] == ' '))
1494                         ||
1495                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1496                          (value[8] == '\0' || value[8] == ' '))) {
1497                     if (request->flags.connectionAuthDisabled) {
1498                         hdr->delAt(pos, connection_auth_blocked);
1499                         continue;
1500                     }
1501                     request->flags.mustKeepalive = true;
1502                     if (!request->flags.accelerated && !request->flags.intercepted) {
1503                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1504                         /*
1505                           We send "Connection: Proxy-Support" header to mark
1506                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1507                           understand the semantics of this header. The RFC should have included
1508                           this recommendation.
1509                         */
1510                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1511                     }
1512                     break;
1513                 }
1514             }
1515         }
1516
1517         if (connection_auth_blocked)
1518             hdr->refreshMask();
1519     }
1520
1521 #if USE_AUTH
1522     /* Handle authentication headers */
1523     if (http->logType.oldType == LOG_TCP_DENIED &&
1524             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1525               reply->sline.status() == Http::scUnauthorized)
1526        ) {
1527         /* Add authentication header */
1528         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1529          * depends on authenticate behaviour: all schemes to date send no extra
1530          * data on 407/401 responses, and do not check the accel state on 401/407
1531          * responses
1532          */
1533         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1534     } else if (request->auth_user_request != NULL)
1535         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1536 #endif
1537
1538     /* Append X-Cache */
1539     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1540                       is_hit ? "HIT" : "MISS", getMyHostname());
1541
1542 #if USE_CACHE_DIGESTS
1543     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1544     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1545                       lookup_type ? lookup_type : "NONE",
1546                       getMyHostname(), getMyPort());
1547
1548 #endif
1549
1550     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1551                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1552                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1553
1554     /* Check whether we should send keep-alive */
1555     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1556         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1557         request->flags.proxyKeepalive = false;
1558     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1559         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1560         request->flags.proxyKeepalive = false;
1561     } else if (request->flags.proxyKeepalive && shutting_down) {
1562         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1563         request->flags.proxyKeepalive = false;
1564     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1565         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1566         request->flags.proxyKeepalive = false;
1567     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1568         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1569         request->flags.proxyKeepalive = false;
1570     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1571         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1572         request->flags.proxyKeepalive = false;
1573     } else if (request->flags.sslBumped && !reply->persistent()) {
1574         // We do not really have to close, but we pretend we are a tunnel.
1575         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1576         request->flags.proxyKeepalive = false;
1577     } else if (request->pinnedConnection() && !reply->persistent()) {
1578         // The peer wants to close the pinned connection
1579         debugs(88, 3, "pinned reply forces close");
1580         request->flags.proxyKeepalive = false;
1581     } else if (http->getConn()) {
1582         ConnStateData * conn = http->getConn();
1583         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1584             // The listening port closed because of a reconfigure
1585             debugs(88, 3, "listening port closed");
1586             request->flags.proxyKeepalive = false;
1587         }
1588     }
1589
1590     // Decide if we send chunked reply
1591     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1592         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1593         request->flags.chunkedReply = true;
1594         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1595     }
1596
1597     /* Append VIA */
1598     if (Config.onoff.via) {
1599         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1600         String strVia;
1601         hdr->getList(Http::HdrType::VIA, &strVia);
1602         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1603                  reply->sline.version.major,
1604                  reply->sline.version.minor,
1605                  ThisCache);
1606         strListAdd(&strVia, bbuf, ',');
1607         hdr->delById(Http::HdrType::VIA);
1608         hdr->putStr(Http::HdrType::VIA, strVia.termedBuf());
1609     }
1610     /* Signal keep-alive or close explicitly */
1611     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1612
1613 #if ADD_X_REQUEST_URI
1614     /*
1615      * Knowing the URI of the request is useful when debugging persistent
1616      * connections in a client; we cannot guarantee the order of http headers,
1617      * but X-Request-URI is likely to be the very last header to ease use from a
1618      * debugger [hdr->entries.count-1].
1619      */
1620     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1621                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1622
1623 #endif
1624
1625     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1626     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1627         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1628             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1629         }
1630         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1631     }
1632
1633     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1634 }
1635
1636 void
1637 clientReplyContext::cloneReply()
1638 {
1639     assert(reply == NULL);
1640
1641     reply = http->storeEntry()->getReply()->clone();
1642     HTTPMSGLOCK(reply);
1643
1644     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1645         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1646         reply->sline.version = Http::ProtocolVersion();
1647     }
1648
1649     /* do header conversions */
1650     buildReplyHeader();
1651 }
1652
1653 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1654 /// We cannot just ignore the entry because it may be locking or otherwise
1655 /// holding an associated cache resource of some sort.
1656 void
1657 clientReplyContext::forgetHit()
1658 {
1659     StoreEntry *e = http->storeEntry();
1660     assert(e); // or we are not dealing with a hit
1661     // We probably have not locked the entry earlier, unfortunately. We lock it
1662     // now so that we can unlock two lines later (and trigger cleanup).
1663     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1664     // used so inconsistently that simply adding locking there leads to bugs.
1665     e->lock("clientReplyContext::forgetHit");
1666     http->storeEntry(NULL);
1667     e->unlock("clientReplyContext::forgetHit"); // may delete e
1668 }
1669
1670 void
1671 clientReplyContext::identifyStoreObject()
1672 {
1673     HttpRequest *r = http->request;
1674
1675     // client sent CC:no-cache or some other condition has been
1676     // encountered which prevents delivering a public/cached object.
1677     if (!r->flags.noCache || r->flags.internal) {
1678         lookingforstore = 5;
1679         StoreEntry::getPublicByRequest (this, r);
1680     } else {
1681         identifyFoundObject (NullStoreEntry::getInstance());
1682     }
1683 }
1684
1685 /**
1686  * Check state of the current StoreEntry object.
1687  * to see if we can determine the final status of the request.
1688  */
1689 void
1690 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1691 {
1692     StoreEntry *e = newEntry;
1693     HttpRequest *r = http->request;
1694
1695     /** \li If the entry received isNull() then we ignore it. */
1696     if (e->isNull()) {
1697         http->storeEntry(NULL);
1698     } else {
1699         http->storeEntry(e);
1700     }
1701
1702     e = http->storeEntry();
1703
1704     /* Release IP-cache entries on reload */
1705     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1706       * 'invalidate' the cached IP entries for this request ???
1707       */
1708     if (r->flags.noCache || r->flags.noCacheHack())
1709         ipcacheInvalidateNegative(r->url.host());
1710
1711 #if USE_CACHE_DIGESTS
1712     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1713 #endif
1714
1715     if (NULL == http->storeEntry()) {
1716         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1717         debugs(85, 3, "StoreEntry is NULL -  MISS");
1718         http->logType = LOG_TCP_MISS;
1719         doGetMoreData();
1720         return;
1721     }
1722
1723     if (Config.onoff.offline) {
1724         /** \li If we are running in offline mode set to HIT */
1725         debugs(85, 3, "offline HIT " << *e);
1726         http->logType = LOG_TCP_HIT;
1727         doGetMoreData();
1728         return;
1729     }
1730
1731     if (http->redirect.status) {
1732         /** \li If redirection status is True force this to be a MISS */
1733         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1734         forgetHit();
1735         http->logType = LOG_TCP_REDIRECT;
1736         doGetMoreData();
1737         return;
1738     }
1739
1740     if (!e->validToSend()) {
1741         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1742         forgetHit();
1743         http->logType = LOG_TCP_MISS;
1744         doGetMoreData();
1745         return;
1746     }
1747
1748     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1749         /* \li Special entries are always hits, no matter what the client says */
1750         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1751         http->logType = LOG_TCP_HIT;
1752         doGetMoreData();
1753         return;
1754     }
1755
1756     if (r->flags.noCache) {
1757         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1758         forgetHit();
1759         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1760         doGetMoreData();
1761         return;
1762     }
1763
1764     debugs(85, 3, "default HIT " << *e);
1765     http->logType = LOG_TCP_HIT;
1766     doGetMoreData();
1767 }
1768
1769 /**
1770  * Request more data from the store for the client Stream
1771  * This is *the* entry point to this module.
1772  *
1773  * Preconditions:
1774  *  - This is the head of the list.
1775  *  - There is at least one more node.
1776  *  - Data context is not null
1777  */
1778 void
1779 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1780 {
1781     /* Test preconditions */
1782     assert(aNode != NULL);
1783     assert(cbdataReferenceValid(aNode));
1784     assert(aNode->node.prev == NULL);
1785     assert(aNode->node.next != NULL);
1786     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1787     assert (context);
1788     assert(context->http == http);
1789
1790     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1791
1792     if (!context->ourNode)
1793         context->ourNode = aNode;
1794
1795     /* no cbdatareference, this is only used once, and safely */
1796     if (context->flags.storelogiccomplete) {
1797         StoreIOBuffer tempBuffer;
1798         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1799         tempBuffer.length = next->readBuffer.length;
1800         tempBuffer.data = next->readBuffer.data;
1801
1802         storeClientCopy(context->sc, http->storeEntry(),
1803                         tempBuffer, clientReplyContext::SendMoreData, context);
1804         return;
1805     }
1806
1807     if (context->http->request->method == Http::METHOD_PURGE) {
1808         context->purgeRequest();
1809         return;
1810     }
1811
1812     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1813
1814     if (context->http->request->method == Http::METHOD_TRACE) {
1815         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1816             context->traceReply(aNode);
1817             return;
1818         }
1819
1820         /* continue forwarding, not finished yet. */
1821         http->logType = LOG_TCP_MISS;
1822
1823         context->doGetMoreData();
1824     } else
1825         context->identifyStoreObject();
1826 }
1827
1828 void
1829 clientReplyContext::doGetMoreData()
1830 {
1831     /* We still have to do store logic processing - vary, cache hit etc */
1832     if (http->storeEntry() != NULL) {
1833         /* someone found the object in the cache for us */
1834         StoreIOBuffer localTempBuffer;
1835
1836         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1837
1838         MemObject *mem_obj = http->storeEntry()->makeMemObject();
1839         if (!mem_obj->hasUris()) {
1840             /*
1841              * This if-block exists because we don't want to clobber
1842              * a preexiting mem_obj->method value if the mem_obj
1843              * already exists.  For example, when a HEAD request
1844              * is a cache hit for a GET response, we want to keep
1845              * the method as GET.
1846              */
1847             mem_obj->setUris(storeId(), http->log_uri, http->request->method);
1848             /**
1849              * Here we can see if the object was
1850              * created using URL or alternative StoreID from helper.
1851              */
1852             debugs(88, 3, "storeId: " << http->storeEntry()->mem_obj->storeId());
1853         }
1854
1855         sc = storeClientListAdd(http->storeEntry(), this);
1856 #if USE_DELAY_POOLS
1857         sc->setDelayId(DelayId::DelayClient(http));
1858 #endif
1859
1860         assert(http->logType.oldType == LOG_TCP_HIT);
1861         reqofs = 0;
1862         /* guarantee nothing has been sent yet! */
1863         assert(http->out.size == 0);
1864         assert(http->out.offset == 0);
1865
1866         if (Ip::Qos::TheConfig.isHitTosActive()) {
1867             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1868         }
1869
1870         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1871             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1872         }
1873
1874         localTempBuffer.offset = reqofs;
1875         localTempBuffer.length = getNextNode()->readBuffer.length;
1876         localTempBuffer.data = getNextNode()->readBuffer.data;
1877         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1878     } else {
1879         /* MISS CASE, http->logType is already set! */
1880         processMiss();
1881     }
1882 }
1883
1884 /** The next node has removed itself from the stream. */
1885 void
1886 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1887 {
1888     /** detach from the stream */
1889     clientStreamDetach(node, http);
1890 }
1891
1892 /**
1893  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1894  * such, writes processed message to the message recipient
1895  */
1896 void
1897 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1898 {
1899     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1900     context->sendMoreData (result);
1901 }
1902
1903 void
1904 clientReplyContext::makeThisHead()
1905 {
1906     /* At least, I think thats what this does */
1907     dlinkDelete(&http->active, &ClientActiveRequests);
1908     dlinkAdd(http, &http->active, &ClientActiveRequests);
1909 }
1910
1911 bool
1912 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1913 {
1914     return /* aborted request */
1915         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1916         /* Upstream read error */ (result.flags.error) ||
1917         /* Upstream EOF */ (sizeToProcess == 0);
1918 }
1919
1920 void
1921 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1922 {
1923     /** call clientWriteComplete so the client socket gets closed
1924      *
1925      * We call into the stream, because we don't know that there is a
1926      * client socket!
1927      */
1928     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1929     StoreIOBuffer localTempBuffer;
1930     flags.complete = 1;
1931     http->request->flags.streamError = true;
1932     localTempBuffer.flags.error = result.flags.error;
1933     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1934                          localTempBuffer);
1935 }
1936
1937 void
1938 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1939 {
1940     StoreIOBuffer localTempBuffer;
1941
1942     if (result.length == 0) {
1943         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1944         flags.complete = 1;
1945     }
1946
1947     assert(result.offset - headers_sz == next()->readBuffer.offset);
1948     localTempBuffer.offset = result.offset - headers_sz;
1949     localTempBuffer.length = result.length;
1950
1951     if (localTempBuffer.length)
1952         localTempBuffer.data = source;
1953
1954     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1955                          localTempBuffer);
1956 }
1957
1958 clientStreamNode *
1959 clientReplyContext::next() const
1960 {
1961     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1962     return getNextNode();
1963 }
1964
1965 void
1966 clientReplyContext::sendBodyTooLargeError()
1967 {
1968     Ip::Address tmp_noaddr;
1969     tmp_noaddr.setNoAddr(); // TODO: make a global const
1970     http->logType = LOG_TCP_DENIED_REPLY;
1971     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1972                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1973                                        http->request);
1974     removeClientStoreReference(&(sc), http);
1975     HTTPMSGUNLOCK(reply);
1976     startError(err);
1977
1978 }
1979
1980 /// send 412 (Precondition Failed) to client
1981 void
1982 clientReplyContext::sendPreconditionFailedError()
1983 {
1984     http->logType = LOG_TCP_HIT;
1985     ErrorState *const err =
1986         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1987                          NULL, http->getConn()->clientConnection->remote, http->request);
1988     removeClientStoreReference(&sc, http);
1989     HTTPMSGUNLOCK(reply);
1990     startError(err);
1991 }
1992
1993 /// send 304 (Not Modified) to client
1994 void
1995 clientReplyContext::sendNotModified()
1996 {
1997     StoreEntry *e = http->storeEntry();
1998     const time_t timestamp = e->timestamp;
1999     HttpReply *const temprep = e->getReply()->make304();
2000     // log as TCP_INM_HIT if code 304 generated for
2001     // If-None-Match request
2002     if (!http->request->flags.ims)
2003         http->logType = LOG_TCP_INM_HIT;
2004     else
2005         http->logType = LOG_TCP_IMS_HIT;
2006     removeClientStoreReference(&sc, http);
2007     createStoreEntry(http->request->method, RequestFlags());
2008     e = http->storeEntry();
2009     // Copy timestamp from the original entry so the 304
2010     // reply has a meaningful Age: header.
2011     e->timestampsSet();
2012     e->timestamp = timestamp;
2013     e->replaceHttpReply(temprep);
2014     e->complete();
2015     /*
2016      * TODO: why put this in the store and then serialise it and
2017      * then parse it again. Simply mark the request complete in
2018      * our context and write the reply struct to the client side.
2019      */
2020     triggerInitialStoreRead();
2021 }
2022
2023 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
2024 /// depending on request method
2025 void
2026 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
2027 {
2028     if (http->request->method == Http::METHOD_GET ||
2029             http->request->method == Http::METHOD_HEAD)
2030         sendNotModified();
2031     else
2032         sendPreconditionFailedError();
2033 }
2034
2035 void
2036 clientReplyContext::processReplyAccess ()
2037 {
2038     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
2039     assert(reply);
2040
2041     /** Don't block our own responses or HTTP status messages */
2042     if (http->logType.oldType == LOG_TCP_DENIED ||
2043             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
2044             alwaysAllowResponse(reply->sline.status())) {
2045         headers_sz = reply->hdr_sz;
2046         processReplyAccessResult(ACCESS_ALLOWED);
2047         return;
2048     }
2049
2050     /** Check for reply to big error */
2051     if (reply->expectedBodyTooLarge(*http->request)) {
2052         sendBodyTooLargeError();
2053         return;
2054     }
2055
2056     headers_sz = reply->hdr_sz;
2057
2058     /** check for absent access controls (permit by default) */
2059     if (!Config.accessList.reply) {
2060         processReplyAccessResult(ACCESS_ALLOWED);
2061         return;
2062     }
2063
2064     /** Process http_reply_access lists */
2065     ACLFilledChecklist *replyChecklist =
2066         clientAclChecklistCreate(Config.accessList.reply, http);
2067     replyChecklist->reply = reply;
2068     HTTPMSGLOCK(replyChecklist->reply);
2069     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2070 }
2071
2072 void
2073 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2074 {
2075     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2076     me->processReplyAccessResult(rv);
2077 }
2078
2079 void
2080 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2081 {
2082     debugs(88, 2, "The reply for " << http->request->method
2083            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2084            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2085
2086     if (accessAllowed != ACCESS_ALLOWED) {
2087         ErrorState *err;
2088         err_type page_id;
2089         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2090
2091         http->logType = LOG_TCP_DENIED_REPLY;
2092
2093         if (page_id == ERR_NONE)
2094             page_id = ERR_ACCESS_DENIED;
2095
2096         Ip::Address tmp_noaddr;
2097         tmp_noaddr.setNoAddr();
2098         err = clientBuildError(page_id, Http::scForbidden, NULL,
2099                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2100                                http->request);
2101
2102         removeClientStoreReference(&sc, http);
2103
2104         HTTPMSGUNLOCK(reply);
2105
2106         startError(err);
2107
2108         return;
2109     }
2110
2111     /* Ok, the reply is allowed, */
2112     http->loggingEntry(http->storeEntry());
2113
2114     ssize_t body_size = reqofs - reply->hdr_sz;
2115     if (body_size < 0) {
2116         reqofs = reply->hdr_sz;
2117         body_size = 0;
2118     }
2119
2120     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2121            (int) body_size << " bytes after " << reply->hdr_sz <<
2122            " bytes of headers");
2123
2124 #if USE_SQUID_ESI
2125
2126     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2127             !alwaysAllowResponse(reply->sline.status()) &&
2128             esiEnableProcessing(reply)) {
2129         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2130         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2131                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2132     }
2133
2134 #endif
2135
2136     if (http->request->method == Http::METHOD_HEAD) {
2137         /* do not forward body for HEAD replies */
2138         body_size = 0;
2139         http->flags.done_copying = true;
2140         flags.complete = 1;
2141     }
2142
2143     assert (!flags.headersSent);
2144     flags.headersSent = true;
2145
2146     StoreIOBuffer localTempBuffer;
2147     char *buf = next()->readBuffer.data;
2148     char *body_buf = buf + reply->hdr_sz;
2149
2150     //Server side may disable ranges under some circumstances.
2151
2152     if ((!http->request->range))
2153         next()->readBuffer.offset = 0;
2154
2155     body_buf -= next()->readBuffer.offset;
2156
2157     if (next()->readBuffer.offset != 0) {
2158         if (next()->readBuffer.offset > body_size) {
2159             /* Can't use any of the body we received. send nothing */
2160             localTempBuffer.length = 0;
2161             localTempBuffer.data = NULL;
2162         } else {
2163             localTempBuffer.length = body_size - next()->readBuffer.offset;
2164             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2165         }
2166     } else {
2167         localTempBuffer.length = body_size;
2168         localTempBuffer.data = body_buf;
2169     }
2170
2171     /* TODO??: move the data in the buffer back by the request header size */
2172     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2173                          http, reply, localTempBuffer);
2174
2175     return;
2176 }
2177
2178 void
2179 clientReplyContext::sendMoreData (StoreIOBuffer result)
2180 {
2181     if (deleting)
2182         return;
2183
2184     StoreEntry *entry = http->storeEntry();
2185
2186     if (ConnStateData * conn = http->getConn()) {
2187         if (!conn->isOpen()) {
2188             debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2189             return;
2190         }
2191         if (conn->pinning.zeroReply) {
2192             debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2193             return;
2194         }
2195
2196         if (reqofs==0 && !http->logType.isTcpHit()) {
2197             if (Ip::Qos::TheConfig.isHitTosActive()) {
2198                 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2199             }
2200             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2201                 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2202             }
2203         }
2204
2205         debugs(88, 5, conn->clientConnection <<
2206                " '" << entry->url() << "'" <<
2207                " out.offset=" << http->out.offset);
2208     }
2209
2210     char *buf = next()->readBuffer.data;
2211
2212     if (buf != result.data) {
2213         /* we've got to copy some data */
2214         assert(result.length <= next()->readBuffer.length);
2215         memcpy(buf, result.data, result.length);
2216     }
2217
2218     /* We've got the final data to start pushing... */
2219     flags.storelogiccomplete = 1;
2220
2221     reqofs += result.length;
2222
2223     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2224
2225     assert(http->request != NULL);
2226
2227     /* ESI TODO: remove this assert once everything is stable */
2228     assert(http->client_stream.head->data
2229            && cbdataReferenceValid(http->client_stream.head->data));
2230
2231     makeThisHead();
2232
2233     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2234            reqofs << " bytes (" << result.length <<
2235            " new bytes)");
2236
2237     /* update size of the request */
2238     reqsize = reqofs;
2239
2240     if (errorInStream(result, reqofs)) {
2241         sendStreamError(result);
2242         return;
2243     }
2244
2245     if (flags.headersSent) {
2246         pushStreamData (result, buf);
2247         return;
2248     }
2249
2250     cloneReply();
2251
2252 #if USE_DELAY_POOLS
2253     if (sc)
2254         sc->setDelayId(DelayId::DelayClient(http,reply));
2255 #endif
2256
2257     /* handle headers */
2258
2259     if (Config.onoff.log_mime_hdrs) {
2260         size_t k;
2261
2262         if ((k = headersEnd(buf, reqofs))) {
2263             safe_free(http->al->headers.reply);
2264             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2265             xstrncpy(http->al->headers.reply, buf, k);
2266         }
2267     }
2268
2269     holdingBuffer = result;
2270     processReplyAccess();
2271     return;
2272 }
2273
2274 /* Using this breaks the client layering just a little!
2275  */
2276 void
2277 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2278 {
2279     assert(http != NULL);
2280     /*
2281      * For erroneous requests, we might not have a h->request,
2282      * so make a fake one.
2283      */
2284
2285     if (http->request == NULL) {
2286         http->request = new HttpRequest(m, AnyP::PROTO_NONE, "http", null_string);
2287         HTTPMSGLOCK(http->request);
2288     }
2289
2290     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2291
2292     // Make entry collapsable ASAP, to increase collapsing chances for others,
2293     // TODO: every must-revalidate and similar request MUST reach the origin,
2294     // but do we have to prohibit others from collapsing on that request?
2295     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2296             !reqFlags.needValidation &&
2297             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2298         // make the entry available for future requests now
2299         Store::Root().allowCollapsing(e, reqFlags, m);
2300     }
2301
2302     sc = storeClientListAdd(e, this);
2303
2304 #if USE_DELAY_POOLS
2305     sc->setDelayId(DelayId::DelayClient(http));
2306 #endif
2307
2308     reqofs = 0;
2309
2310     reqsize = 0;
2311
2312     /* I don't think this is actually needed! -- adrian */
2313     /* http->reqbuf = http->norm_reqbuf; */
2314     //    assert(http->reqbuf == http->norm_reqbuf);
2315     /* The next line is illegal because we don't know if the client stream
2316      * buffers have been set up
2317      */
2318     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2319     //        SendMoreData, this);
2320     /* So, we mark the store logic as complete */
2321     flags.storelogiccomplete = 1;
2322
2323     /* and get the caller to request a read, from whereever they are */
2324     /* NOTE: after ANY data flows down the pipe, even one step,
2325      * this function CAN NOT be used to manage errors
2326      */
2327     http->storeEntry(e);
2328 }
2329
2330 ErrorState *
2331 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2332                  Ip::Address &src_addr, HttpRequest * request)
2333 {
2334     ErrorState *err = new ErrorState(page_id, status, request);
2335     err->src_addr = src_addr;
2336
2337     if (url)
2338         err->url = xstrdup(url);
2339
2340     return err;
2341 }
2342