src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "globals.h"
  24 #include "http/Stream.h"
  25 #include "HttpHeaderTools.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "ip/QosConfig.h"
  29 #include "ipcache.h"
  30 #include "log/access_log.h"
  31 #include "MemObject.h"
  32 #include "mime_header.h"
  33 #include "neighbors.h"
  34 #include "refresh.h"
  35 #include "RequestFlags.h"
  36 #include "SquidConfig.h"
  37 #include "SquidTime.h"
  38 #include "Store.h"
  39 #include "StrList.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #if USE_AUTH
  43 #include "auth/UserRequest.h"
  44 #endif
  45 #if USE_DELAY_POOLS
  46 #include "DelayPools.h"
  47 #endif
  48 #if USE_SQUID_ESI
  49 #include "esi/Esi.h"
  50 #endif
  51
  52 #include <memory>
  53
  54 CBDATA_CLASS_INIT(clientReplyContext);
  55
  56 /* Local functions */
  57 extern "C" CSS clientReplyStatus;
  58 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  59
  60 /* privates */
  61
  62 clientReplyContext::~clientReplyContext()
  63 {
  64     deleting = true;
  65     /* This may trigger a callback back into SendMoreData as the cbdata
  66      * is still valid
  67      */
  68     removeClientStoreReference(&sc, http);
  69     /* old_entry might still be set if we didn't yet get the reply
  70      * code in HandleIMSReply() */
  71     removeStoreReference(&old_sc, &old_entry);
  72     safe_free(tempBuffer.data);
  73     cbdataReferenceDone(http);
  74     HTTPMSGUNLOCK(reply);
  75 }
  76
  77 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  78     purgeStatus(Http::scNone),
  79     lookingforstore(0),
  80     http(cbdataReference(clientContext)),
  81     headers_sz(0),
  82     sc(NULL),
  83     old_reqsize(0),
  84     reqsize(0),
  85     reqofs(0),
  86 #if USE_CACHE_DIGESTS
  87     lookup_type(NULL),
  88 #endif
  89     ourNode(NULL),
  90     reply(NULL),
  91     old_entry(NULL),
  92     old_sc(NULL),
  93     deleting(false)
  94 {
  95     *tempbuf = 0;
  96 }
  97
  98 /** Create an error in the store awaiting the client side to read it.
  99  *
 100  * This may be better placed in the clientStream logic, but it has not been
 101  * relocated there yet
 102  */
 103 void
 104 clientReplyContext::setReplyToError(
 105     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 106     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 107 #if USE_AUTH
 108     Auth::UserRequest::Pointer auth_user_request
 109 #else
 110     void*
 111 #endif
 112 )
 113 {
 114     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 115
 116     if (unparsedrequest)
 117         errstate->request_hdrs = xstrdup(unparsedrequest);
 118
 119 #if USE_AUTH
 120     errstate->auth_user_request = auth_user_request;
 121 #endif
 122     setReplyToError(method, errstate);
 123 }
 124
 125 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 126 {
 127     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 128         /* prevent confusion over whether we default to persistent or not */
 129         http->request->flags.proxyKeepalive = false;
 130
 131     http->al->http.code = errstate->httpStatus;
 132
 133     if (http->request)
 134         http->request->ignoreRange("responding with a Squid-generated error");
 135
 136     createStoreEntry(method, RequestFlags());
 137     assert(errstate->callback_data == NULL);
 138     errorAppendEntry(http->storeEntry(), errstate);
 139     /* Now the caller reads to get this */
 140 }
 141
 142 void
 143 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 144 {
 145     Must(futureReply);
 146     http->al->http.code = futureReply->sline.status();
 147
 148     HttpRequestMethod method;
 149     if (http->request) { // nil on responses to unparsable requests
 150         http->request->ignoreRange("responding with a Squid-generated reply");
 151         method = http->request->method;
 152     }
 153
 154     createStoreEntry(method, RequestFlags());
 155
 156     http->storeEntry()->storeErrorResponse(futureReply);
 157     /* Now the caller reads to get futureReply */
 158 }
 159
 160 // Assumes that the entry contains an error response without Content-Range.
 161 // To use with regular entries, make HTTP Range header removal conditional.
 162 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 163 {
 164     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 165     sc = storeClientListAdd(entry, this);
 166 #if USE_DELAY_POOLS
 167     sc->setDelayId(DelayId::DelayClient(http));
 168 #endif
 169     reqofs = 0;
 170     reqsize = 0;
 171     if (http->request)
 172         http->request->ignoreRange(reason);
 173     flags.storelogiccomplete = 1;
 174     http->storeEntry(entry);
 175 }
 176
 177 void
 178 clientReplyContext::removeStoreReference(store_client ** scp,
 179         StoreEntry ** ep)
 180 {
 181     StoreEntry *e;
 182     store_client *sc_tmp = *scp;
 183
 184     if ((e = *ep) != NULL) {
 185         *ep = NULL;
 186         storeUnregister(sc_tmp, e, this);
 187         *scp = NULL;
 188         e->unlock("clientReplyContext::removeStoreReference");
 189     }
 190 }
 191
 192 void
 193 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 194 {
 195     StoreEntry *reference = aHttpRequest->storeEntry();
 196     removeStoreReference(scp, &reference);
 197     aHttpRequest->storeEntry(reference);
 198 }
 199
 200 void
 201 clientReplyContext::saveState()
 202 {
 203     assert(old_sc == NULL);
 204     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 205     old_entry = http->storeEntry();
 206     old_sc = sc;
 207     old_reqsize = reqsize;
 208     tempBuffer.offset = reqofs;
 209     /* Prevent accessing the now saved entries */
 210     http->storeEntry(NULL);
 211     sc = NULL;
 212     reqsize = 0;
 213     reqofs = 0;
 214 }
 215
 216 void
 217 clientReplyContext::restoreState()
 218 {
 219     assert(old_sc != NULL);
 220     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 221     removeClientStoreReference(&sc, http);
 222     http->storeEntry(old_entry);
 223     sc = old_sc;
 224     reqsize = old_reqsize;
 225     reqofs = tempBuffer.offset;
 226     /* Prevent accessed the old saved entries */
 227     old_entry = NULL;
 228     old_sc = NULL;
 229     old_reqsize = 0;
 230     tempBuffer.offset = 0;
 231 }
 232
 233 void
 234 clientReplyContext::startError(ErrorState * err)
 235 {
 236     createStoreEntry(http->request->method, RequestFlags());
 237     triggerInitialStoreRead();
 238     errorAppendEntry(http->storeEntry(), err);
 239 }
 240
 241 clientStreamNode *
 242 clientReplyContext::getNextNode() const
 243 {
 244     return (clientStreamNode *)ourNode->node.next->data;
 245 }
 246
 247 /* This function is wrong - the client parameters don't include the
 248  * header offset
 249  */
 250 void
 251 clientReplyContext::triggerInitialStoreRead()
 252 {
 253     /* when confident, 0 becomes reqofs, and then this factors into
 254      * startSendProcess
 255      */
 256     assert(reqofs == 0);
 257     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 258     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 259 }
 260
 261 /* there is an expired entry in the store.
 262  * setup a temporary buffer area and perform an IMS to the origin
 263  */
 264 void
 265 clientReplyContext::processExpired()
 266 {
 267     const char *url = storeId();
 268     StoreEntry *entry = NULL;
 269     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 270     assert(http->storeEntry()->lastmod >= 0);
 271     /*
 272      * check if we are allowed to contact other servers
 273      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 274      *      a stale entry *if* it matches client requirements
 275      */
 276
 277     if (http->onlyIfCached()) {
 278         processOnlyIfCachedMiss();
 279         return;
 280     }
 281
 282     http->request->flags.refresh = true;
 283 #if STORE_CLIENT_LIST_DEBUG
 284     /* Prevent a race with the store client memory free routines
 285      */
 286     assert(storeClientIsThisAClient(sc, this));
 287 #endif
 288     /* Prepare to make a new temporary request */
 289     saveState();
 290     entry = storeCreateEntry(url,
 291                              http->log_uri, http->request->flags, http->request->method);
 292     /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 293     sc = storeClientListAdd(entry, this);
 294 #if USE_DELAY_POOLS
 295     /* delay_id is already set on original store client */
 296     sc->setDelayId(DelayId::DelayClient(http));
 297 #endif
 298
 299     http->request->lastmod = old_entry->lastmod;
 300
 301     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 302         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 303         if (old_entry->hasEtag(etag) && !etag.weak)
 304             http->request->etag = etag.str;
 305     }
 306
 307     debugs(88, 5, "clientReplyContext::processExpired : lastmod " << entry->lastmod );
 308     http->storeEntry(entry);
 309     assert(http->out.offset == 0);
 310     assert(http->request->clientConnectionManager == http->getConn());
 311
 312     /*
 313      * A refcounted pointer so that FwdState stays around as long as
 314      * this clientReplyContext does
 315      */
 316     Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 317     FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 318
 319     /* Register with storage manager to receive updates when data comes in. */
 320
 321     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 322         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 323
 324     {
 325         /* start counting the length from 0 */
 326         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 327         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 328     }
 329 }
 330
 331 void
 332 clientReplyContext::sendClientUpstreamResponse()
 333 {
 334     StoreIOBuffer tempresult;
 335     removeStoreReference(&old_sc, &old_entry);
 336     /* here the data to send is the data we just received */
 337     tempBuffer.offset = 0;
 338     old_reqsize = 0;
 339     /* sendMoreData tracks the offset as well.
 340      * Force it back to zero */
 341     reqofs = 0;
 342     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 343     /* TODO: provide sendMoreData with the ready parsed reply */
 344     tempresult.length = reqsize;
 345     tempresult.data = tempbuf;
 346     sendMoreData(tempresult);
 347 }
 348
 349 void
 350 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 351 {
 352     clientReplyContext *context = (clientReplyContext *)data;
 353     context->handleIMSReply(result);
 354 }
 355
 356 void
 357 clientReplyContext::sendClientOldEntry()
 358 {
 359     /* Get the old request back */
 360     restoreState();
 361     /* here the data to send is in the next nodes buffers already */
 362     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 363     /* sendMoreData tracks the offset as well.
 364      * Force it back to zero */
 365     reqofs = 0;
 366     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 367     sendMoreData(tempresult);
 368 }
 369
 370 /* This is the workhorse of the HandleIMSReply callback.
 371  *
 372  * It is called when we've got data back from the origin following our
 373  * IMS request to revalidate a stale entry.
 374  */
 375 void
 376 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 377 {
 378     if (deleting)
 379         return;
 380
 381     debugs(88, 3, "handleIMSReply: " << http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes" );
 382
 383     if (http->storeEntry() == NULL)
 384         return;
 385
 386     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 387         return;
 388
 389     /* update size of the request */
 390     reqsize = result.length + reqofs;
 391
 392     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 393
 394     // request to origin was aborted
 395     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 396         debugs(88, 3, "handleIMSReply: request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client" );
 397         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 398         sendClientOldEntry();
 399     }
 400
 401     const HttpReply *old_rep = old_entry->getReply();
 402
 403     // origin replied 304
 404     if (status == Http::scNotModified) {
 405         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 406         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 407
 408         // update headers on existing entry
 409         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 410
 411         // if client sent IMS
 412
 413         if (http->request->flags.ims && !old_entry->modifiedSince(http->request)) {
 414             // forward the 304 from origin
 415             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client");
 416             sendClientUpstreamResponse();
 417         } else {
 418             // send existing entry, it's still valid
 419             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " <<
 420                    old_rep->sline.status() << " to client");
 421             sendClientOldEntry();
 422         }
 423     }
 424
 425     // origin replied with a non-error code
 426     else if (status > Http::scNone && status < Http::scInternalServerError) {
 427         // forward response from origin
 428         http->logType = LOG_TCP_REFRESH_MODIFIED;
 429         debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client");
 430         sendClientUpstreamResponse();
 431     }
 432
 433     // origin replied with an error
 434     else if (http->request->flags.failOnValidationError) {
 435         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 436         debugs(88, 3, "handleIMSReply: origin replied with error " << status <<
 437                ", forwarding to client due to fail_on_validation_err");
 438         sendClientUpstreamResponse();
 439     } else {
 440         // ignore and let client have old entry
 441         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 442         debugs(88, 3, "handleIMSReply: origin replied with error " <<
 443                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 444         sendClientOldEntry();
 445     }
 446 }
 447
 448 SQUIDCEXTERN CSR clientGetMoreData;
 449 SQUIDCEXTERN CSD clientReplyDetach;
 450
 451 /**
 452  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 453  * have been parsed.  Normally this should be a single call, but
 454  * it might take more than one.  As soon as we have the headers,
 455  * we hand off to clientSendMoreData, processExpired, or
 456  * processMiss.
 457  */
 458 void
 459 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 460 {
 461     clientReplyContext *context = (clientReplyContext *)data;
 462     context->cacheHit(result);
 463 }
 464
 465 /**
 466  * Process a possible cache HIT.
 467  */
 468 void
 469 clientReplyContext::cacheHit(StoreIOBuffer result)
 470 {
 471     /** Ignore if the HIT object is being deleted. */
 472     if (deleting) {
 473         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 474         return;
 475     }
 476
 477     StoreEntry *e = http->storeEntry();
 478
 479     HttpRequest *r = http->request;
 480
 481     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 482
 483     if (http->storeEntry() == NULL) {
 484         debugs(88, 3, "clientCacheHit: request aborted");
 485         return;
 486     } else if (result.flags.error) {
 487         /* swap in failure */
 488         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 489         http->logType = LOG_TCP_SWAPFAIL_MISS;
 490         removeClientStoreReference(&sc, http);
 491         processMiss();
 492         return;
 493     }
 494
 495     if (result.length == 0) {
 496         debugs(88, 5, "store IO buffer has no content. MISS");
 497         /* the store couldn't get enough data from the file for us to id the
 498          * object
 499          */
 500         /* treat as a miss */
 501         http->logType = LOG_TCP_MISS;
 502         processMiss();
 503         return;
 504     }
 505
 506     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 507     /* update size of the request */
 508     reqsize = result.length + reqofs;
 509
 510     /*
 511      * Got the headers, now grok them
 512      */
 513     assert(http->logType.oldType == LOG_TCP_HIT);
 514
 515     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 516         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 517         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 518         processMiss();
 519         return;
 520     }
 521
 522     switch (varyEvaluateMatch(e, r)) {
 523
 524     case VARY_NONE:
 525         /* No variance detected. Continue as normal */
 526         break;
 527
 528     case VARY_MATCH:
 529         /* This is the correct entity for this request. Continue */
 530         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 531         break;
 532
 533     case VARY_OTHER:
 534         /* This is not the correct entity for this request. We need
 535          * to requery the cache.
 536          */
 537         removeClientStoreReference(&sc, http);
 538         e = NULL;
 539         /* Note: varyEvalyateMatch updates the request with vary information
 540          * so we only get here once. (it also takes care of cancelling loops)
 541          */
 542         debugs(88, 2, "clientProcessHit: Vary detected!");
 543         clientGetMoreData(ourNode, http);
 544         return;
 545
 546     case VARY_CANCEL:
 547         /* varyEvaluateMatch found a object loop. Process as miss */
 548         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 549         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 550         processMiss();
 551         return;
 552     }
 553
 554     if (r->method == Http::METHOD_PURGE) {
 555         debugs(88, 5, "PURGE gets a HIT");
 556         removeClientStoreReference(&sc, http);
 557         e = NULL;
 558         purgeRequest();
 559         return;
 560     }
 561
 562     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 563         debugs(88, 5, "negative-HIT");
 564         http->logType = LOG_TCP_NEGATIVE_HIT;
 565         sendMoreData(result);
 566     } else if (blockedHit()) {
 567         debugs(88, 5, "send_hit forces a MISS");
 568         http->logType = LOG_TCP_MISS;
 569         processMiss();
 570         return;
 571     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 572         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 573         /*
 574          * We hold a stale copy; it needs to be validated
 575          */
 576         /*
 577          * The 'needValidation' flag is used to prevent forwarding
 578          * loops between siblings.  If our copy of the object is stale,
 579          * then we should probably only use parents for the validation
 580          * request.  Otherwise two siblings could generate a loop if
 581          * both have a stale version of the object.
 582          */
 583         r->flags.needValidation = true;
 584
 585         if (e->lastmod < 0) {
 586             debugs(88, 3, "validate HIT object? NO. Missing Last-Modified header. Do MISS.");
 587             /*
 588              * Previous reply didn't have a Last-Modified header,
 589              * we cannot revalidate it.
 590              */
 591             http->logType = LOG_TCP_MISS;
 592             processMiss();
 593         } else if (r->flags.noCache) {
 594             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 595             /*
 596              * This did not match a refresh pattern that overrides no-cache
 597              * we should honour the client no-cache header.
 598              */
 599             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 600             processMiss();
 601         } else if (r->url.getScheme() == AnyP::PROTO_HTTP) {
 602             debugs(88, 3, "validate HIT object? YES.");
 603             /*
 604              * Object needs to be revalidated
 605              * XXX This could apply to FTP as well, if Last-Modified is known.
 606              */
 607             processExpired();
 608         } else {
 609             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 610             /*
 611              * We don't know how to re-validate other protocols. Handle
 612              * them as if the object has expired.
 613              */
 614             http->logType = LOG_TCP_MISS;
 615             processMiss();
 616         }
 617     } else if (r->conditional()) {
 618         debugs(88, 5, "conditional HIT");
 619         processConditional(result);
 620     } else {
 621         /*
 622          * plain ol' cache hit
 623          */
 624         debugs(88, 5, "plain old HIT");
 625
 626 #if USE_DELAY_POOLS
 627         if (e->store_status != STORE_OK)
 628             http->logType = LOG_TCP_MISS;
 629         else
 630 #endif
 631             if (e->mem_status == IN_MEMORY)
 632                 http->logType = LOG_TCP_MEM_HIT;
 633             else if (Config.onoff.offline)
 634                 http->logType = LOG_TCP_OFFLINE_HIT;
 635
 636         sendMoreData(result);
 637     }
 638 }
 639
 640 /**
 641  * Prepare to fetch the object as it's a cache miss of some kind.
 642  */
 643 void
 644 clientReplyContext::processMiss()
 645 {
 646     char *url = http->uri;
 647     HttpRequest *r = http->request;
 648     ErrorState *err = NULL;
 649     debugs(88, 4, r->method << ' ' << url);
 650
 651     /**
 652      * We might have a left-over StoreEntry from a failed cache hit
 653      * or IMS request.
 654      */
 655     if (http->storeEntry()) {
 656         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 657             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 658             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 659             http->storeEntry()->dump(1);
 660         }
 661
 662         removeClientStoreReference(&sc, http);
 663     }
 664
 665     /** Check if its a PURGE request to be actioned. */
 666     if (r->method == Http::METHOD_PURGE) {
 667         purgeRequest();
 668         return;
 669     }
 670
 671     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 672     if (r->method == Http::METHOD_OTHER) {
 673         purgeAllCached();
 674     }
 675
 676     /** Check if 'only-if-cached' flag is set. Action if so. */
 677     if (http->onlyIfCached()) {
 678         processOnlyIfCachedMiss();
 679         return;
 680     }
 681
 682     /// Deny loops
 683     if (r->flags.loopDetected) {
 684         http->al->http.code = Http::scForbidden;
 685         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 686         createStoreEntry(r->method, RequestFlags());
 687         errorAppendEntry(http->storeEntry(), err);
 688         triggerInitialStoreRead();
 689         return;
 690     } else {
 691         assert(http->out.offset == 0);
 692         createStoreEntry(r->method, r->flags);
 693         triggerInitialStoreRead();
 694
 695         if (http->redirect.status) {
 696             HttpReply *rep = new HttpReply;
 697             http->logType = LOG_TCP_REDIRECT;
 698             http->storeEntry()->releaseRequest();
 699             rep->redirect(http->redirect.status, http->redirect.location);
 700             http->storeEntry()->replaceHttpReply(rep);
 701             http->storeEntry()->complete();
 702             return;
 703         }
 704
 705         assert(r->clientConnectionManager == http->getConn());
 706
 707         /** Start forwarding to get the new object from network */
 708         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 709         FwdState::Start(conn, http->storeEntry(), r, http->al);
 710     }
 711 }
 712
 713 /**
 714  * client issued a request with an only-if-cached cache-control directive;
 715  * we did not find a cached object that can be returned without
 716  *     contacting other servers;
 717  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 718  */
 719 void
 720 clientReplyContext::processOnlyIfCachedMiss()
 721 {
 722     debugs(88, 4, http->request->method << ' ' << http->uri);
 723     http->al->http.code = Http::scGatewayTimeout;
 724     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 725                                        http->getConn()->clientConnection->remote, http->request);
 726     removeClientStoreReference(&sc, http);
 727     startError(err);
 728 }
 729
 730 /// process conditional request from client
 731 void
 732 clientReplyContext::processConditional(StoreIOBuffer &result)
 733 {
 734     StoreEntry *const e = http->storeEntry();
 735
 736     if (e->getReply()->sline.status() != Http::scOkay) {
 737         debugs(88, 4, "clientReplyContext::processConditional: Reply code " <<
 738                e->getReply()->sline.status() << " != 200");
 739         http->logType = LOG_TCP_MISS;
 740         processMiss();
 741         return;
 742     }
 743
 744     HttpRequest &r = *http->request;
 745
 746     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 747         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 748         sendPreconditionFailedError();
 749         return;
 750     }
 751
 752     bool matchedIfNoneMatch = false;
 753     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 754         if (!e->hasIfNoneMatchEtag(r)) {
 755             // RFC 2616: ignore IMS if If-None-Match did not match
 756             r.flags.ims = false;
 757             r.ims = -1;
 758             r.imslen = 0;
 759             r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 760             http->logType = LOG_TCP_MISS;
 761             sendMoreData(result);
 762             return;
 763         }
 764
 765         if (!r.flags.ims) {
 766             // RFC 2616: if If-None-Match matched and there is no IMS,
 767             // reply with 304 Not Modified or 412 Precondition Failed
 768             sendNotModifiedOrPreconditionFailedError();
 769             return;
 770         }
 771
 772         // otherwise check IMS below to decide if we reply with 304 or 412
 773         matchedIfNoneMatch = true;
 774     }
 775
 776     if (r.flags.ims) {
 777         // handle If-Modified-Since requests from the client
 778         if (e->modifiedSince(&r)) {
 779             http->logType = LOG_TCP_IMS_HIT;
 780             sendMoreData(result);
 781             return;
 782         }
 783
 784         if (matchedIfNoneMatch) {
 785             // If-None-Match matched, reply with 304 Not Modified or
 786             // 412 Precondition Failed
 787             sendNotModifiedOrPreconditionFailedError();
 788             return;
 789         }
 790
 791         // otherwise reply with 304 Not Modified
 792         sendNotModified();
 793     }
 794 }
 795
 796 /// whether squid.conf send_hit prevents us from serving this hit
 797 bool
 798 clientReplyContext::blockedHit() const
 799 {
 800     if (!Config.accessList.sendHit)
 801         return false; // hits are not blocked by default
 802
 803     if (http->flags.internal)
 804         return false; // internal content "hits" cannot be blocked
 805
 806     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 807         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 808         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 809         HTTPMSGLOCK(chl->reply);
 810         return chl->fastCheck() != ACCESS_ALLOWED; // when in doubt, block
 811     }
 812
 813     // This does not happen, I hope, because we are called from CacheHit, which
 814     // is called via a storeClientCopy() callback, and store should initialize
 815     // the reply before calling that callback.
 816     debugs(88, 3, "Missing reply!");
 817     return false;
 818 }
 819
 820 void
 821 clientReplyContext::purgeRequestFindObjectToPurge()
 822 {
 823     /* Try to find a base entry */
 824     http->flags.purging = true;
 825     lookingforstore = 1;
 826
 827     // TODO: can we use purgeAllCached() here instead of doing the
 828     // getPublicByRequestMethod() dance?
 829     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 830 }
 831
 832 // Purges all entries with a given url
 833 // TODO: move to SideAgent parent, when we have one
 834 /*
 835  * We probably cannot purge Vary-affected responses because their MD5
 836  * keys depend on vary headers.
 837  */
 838 void
 839 purgeEntriesByUrl(HttpRequest * req, const char *url)
 840 {
 841 #if USE_HTCP
 842     bool get_or_head_sent = false;
 843 #endif
 844
 845     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 846         if (m.respMaybeCacheable()) {
 847             if (StoreEntry *entry = storeGetPublic(url, m)) {
 848                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 849 #if USE_HTCP
 850                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 851                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 852                     get_or_head_sent = true;
 853                 }
 854 #endif
 855                 entry->release();
 856             }
 857         }
 858     }
 859
 860 #if USE_HTCP
 861     if (!get_or_head_sent) {
 862         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 863     }
 864 #endif
 865 }
 866
 867 void
 868 clientReplyContext::purgeAllCached()
 869 {
 870     // XXX: performance regression, c_str() reallocates
 871     SBuf url(http->request->effectiveRequestUri());
 872     purgeEntriesByUrl(http->request, url.c_str());
 873 }
 874
 875 void
 876 clientReplyContext::created(StoreEntry *newEntry)
 877 {
 878     if (lookingforstore == 1)
 879         purgeFoundGet(newEntry);
 880     else if (lookingforstore == 2)
 881         purgeFoundHead(newEntry);
 882     else if (lookingforstore == 3)
 883         purgeDoPurgeGet(newEntry);
 884     else if (lookingforstore == 4)
 885         purgeDoPurgeHead(newEntry);
 886     else if (lookingforstore == 5)
 887         identifyFoundObject(newEntry);
 888 }
 889
 890 void
 891 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 892 {
 893     if (newEntry->isNull()) {
 894         lookingforstore = 2;
 895         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 896     } else
 897         purgeFoundObject (newEntry);
 898 }
 899
 900 void
 901 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 902 {
 903     if (newEntry->isNull())
 904         purgeDoMissPurge();
 905     else
 906         purgeFoundObject (newEntry);
 907 }
 908
 909 void
 910 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 911 {
 912     assert (entry && !entry->isNull());
 913
 914     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 915         http->logType = LOG_TCP_DENIED;
 916         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 917                                            http->getConn()->clientConnection->remote, http->request);
 918         startError(err);
 919         return; // XXX: leaking unused entry if some store does not keep it
 920     }
 921
 922     StoreIOBuffer localTempBuffer;
 923     /* Swap in the metadata */
 924     http->storeEntry(entry);
 925
 926     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 927     http->storeEntry()->createMemObject(storeId(), http->log_uri,
 928                                         http->request->method);
 929
 930     sc = storeClientListAdd(http->storeEntry(), this);
 931
 932     http->logType = LOG_TCP_HIT;
 933
 934     reqofs = 0;
 935
 936     localTempBuffer.offset = http->out.offset;
 937
 938     localTempBuffer.length = next()->readBuffer.length;
 939
 940     localTempBuffer.data = next()->readBuffer.data;
 941
 942     storeClientCopy(sc, http->storeEntry(),
 943                     localTempBuffer, CacheHit, this);
 944 }
 945
 946 void
 947 clientReplyContext::purgeRequest()
 948 {
 949     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 950            Config2.onoff.enable_purge);
 951
 952     if (!Config2.onoff.enable_purge) {
 953         http->logType = LOG_TCP_DENIED;
 954         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL, http->getConn()->clientConnection->remote, http->request);
 955         startError(err);
 956         return;
 957     }
 958
 959     /* Release both IP cache */
 960     ipcacheInvalidate(http->request->url.host());
 961
 962     if (!http->flags.purging)
 963         purgeRequestFindObjectToPurge();
 964     else
 965         purgeDoMissPurge();
 966 }
 967
 968 void
 969 clientReplyContext::purgeDoMissPurge()
 970 {
 971     http->logType = LOG_TCP_MISS;
 972     lookingforstore = 3;
 973     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
 974 }
 975
 976 void
 977 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
 978 {
 979     assert (newEntry);
 980     /* Move to new() when that is created */
 981     purgeStatus = Http::scNotFound;
 982
 983     if (!newEntry->isNull()) {
 984         /* Release the cached URI */
 985         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
 986 #if USE_HTCP
 987         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
 988 #endif
 989         newEntry->release();
 990         purgeStatus = Http::scOkay;
 991     }
 992
 993     lookingforstore = 4;
 994     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 995 }
 996
 997 void
 998 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
 999 {
1000     if (newEntry && !newEntry->isNull()) {
1001         debugs(88, 4, "HEAD " << newEntry->url());
1002 #if USE_HTCP
1003         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1004 #endif
1005         newEntry->release();
1006         purgeStatus = Http::scOkay;
1007     }
1008
1009     /* And for Vary, release the base URI if none of the headers was included in the request */
1010     if (!http->request->vary_headers.isEmpty()
1011             && http->request->vary_headers.find('=') != SBuf::npos) {
1012         // XXX: performance regression, c_str() reallocates
1013         SBuf tmp(http->request->effectiveRequestUri());
1014         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1015
1016         if (entry) {
1017             debugs(88, 4, "Vary GET " << entry->url());
1018 #if USE_HTCP
1019             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1020 #endif
1021             entry->release();
1022             purgeStatus = Http::scOkay;
1023         }
1024
1025         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1026
1027         if (entry) {
1028             debugs(88, 4, "Vary HEAD " << entry->url());
1029 #if USE_HTCP
1030             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1031 #endif
1032             entry->release();
1033             purgeStatus = Http::scOkay;
1034         }
1035     }
1036
1037     /*
1038      * Make a new entry to hold the reply to be written
1039      * to the client.
1040      */
1041     /* FIXME: This doesn't need to go through the store. Simply
1042      * push down the client chain
1043      */
1044     createStoreEntry(http->request->method, RequestFlags());
1045
1046     triggerInitialStoreRead();
1047
1048     HttpReply *rep = new HttpReply;
1049     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1050     http->storeEntry()->replaceHttpReply(rep);
1051     http->storeEntry()->complete();
1052 }
1053
1054 void
1055 clientReplyContext::traceReply(clientStreamNode * node)
1056 {
1057     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1058     StoreIOBuffer localTempBuffer;
1059     createStoreEntry(http->request->method, RequestFlags());
1060     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1061     localTempBuffer.length = nextNode->readBuffer.length;
1062     localTempBuffer.data = nextNode->readBuffer.data;
1063     storeClientCopy(sc, http->storeEntry(),
1064                     localTempBuffer, SendMoreData, this);
1065     http->storeEntry()->releaseRequest();
1066     http->storeEntry()->buffer();
1067     HttpReply *rep = new HttpReply;
1068     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1069     http->storeEntry()->replaceHttpReply(rep);
1070     http->request->swapOut(http->storeEntry());
1071     http->storeEntry()->complete();
1072 }
1073
1074 #define SENDING_BODY 0
1075 #define SENDING_HDRSONLY 1
1076 int
1077 clientReplyContext::checkTransferDone()
1078 {
1079     StoreEntry *entry = http->storeEntry();
1080
1081     if (entry == NULL)
1082         return 0;
1083
1084     /*
1085      * For now, 'done_copying' is used for special cases like
1086      * Range and HEAD requests.
1087      */
1088     if (http->flags.done_copying)
1089         return 1;
1090
1091     if (http->request->flags.chunkedReply && !flags.complete) {
1092         // last-chunk was not sent
1093         return 0;
1094     }
1095
1096     /*
1097      * Handle STORE_OK objects.
1098      * objectLen(entry) will be set proprely.
1099      * RC: Does objectLen(entry) include the Headers?
1100      * RC: Yes.
1101      */
1102     if (entry->store_status == STORE_OK) {
1103         return storeOKTransferDone();
1104     } else {
1105         return storeNotOKTransferDone();
1106     }
1107 }
1108
1109 int
1110 clientReplyContext::storeOKTransferDone() const
1111 {
1112     assert(http->storeEntry()->objectLen() >= 0);
1113     assert(http->storeEntry()->objectLen() >= headers_sz);
1114     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1115         debugs(88,3,HERE << "storeOKTransferDone " <<
1116                " out.offset=" << http->out.offset <<
1117                " objectLen()=" << http->storeEntry()->objectLen() <<
1118                " headers_sz=" << headers_sz);
1119         return 1;
1120     }
1121
1122     return 0;
1123 }
1124
1125 int
1126 clientReplyContext::storeNotOKTransferDone() const
1127 {
1128     /*
1129      * Now, handle STORE_PENDING objects
1130      */
1131     MemObject *mem = http->storeEntry()->mem_obj;
1132     assert(mem != NULL);
1133     assert(http->request != NULL);
1134     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1135     HttpReply const *curReply = mem->getReply();
1136
1137     if (headers_sz == 0)
1138         /* haven't found end of headers yet */
1139         return 0;
1140
1141     /*
1142      * Figure out how much data we are supposed to send.
1143      * If we are sending a body and we don't have a content-length,
1144      * then we must wait for the object to become STORE_OK.
1145      */
1146     if (curReply->content_length < 0)
1147         return 0;
1148
1149     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1150
1151     if (http->out.size < expectedLength)
1152         return 0;
1153     else {
1154         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1155                " out.size=" << http->out.size <<
1156                " expectedLength=" << expectedLength);
1157         return 1;
1158     }
1159 }
1160
1161 /* A write has completed, what is the next status based on the
1162  * canonical request data?
1163  * 1 something is wrong
1164  * 0 nothing is wrong.
1165  *
1166  */
1167 int
1168 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1169 {
1170 #if SIZEOF_INT64_T == 4
1171     if (http->out.size > 0x7FFF0000) {
1172         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1173         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1174         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1175         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1176         return 1;
1177     }
1178
1179     if (http->out.offset > 0x7FFF0000) {
1180         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1181         debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1182         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1183         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1184         return 1;
1185     }
1186
1187 #endif
1188     return 0;
1189 }
1190
1191 /* Preconditions:
1192  * *http is a valid structure.
1193  * fd is either -1, or an open fd.
1194  *
1195  * TODO: enumify this
1196  *
1197  * This function is used by any http request sink, to determine the status
1198  * of the object.
1199  */
1200 clientStream_status_t
1201 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1202 {
1203     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1204     assert (context);
1205     assert (context->http == http);
1206     return context->replyStatus();
1207 }
1208
1209 clientStream_status_t
1210 clientReplyContext::replyStatus()
1211 {
1212     int done;
1213     /* Here because lower nodes don't need it */
1214
1215     if (http->storeEntry() == NULL) {
1216         debugs(88, 5, "clientReplyStatus: no storeEntry");
1217         return STREAM_FAILED;   /* yuck, but what can we do? */
1218     }
1219
1220     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1221         /* TODO: Could upstream read errors (result.flags.error) be
1222          * lost, and result in undersize requests being considered
1223          * complete. Should we tcp reset such connections ?
1224          */
1225         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1226         return STREAM_FAILED;
1227     }
1228
1229     if ((done = checkTransferDone()) != 0 || flags.complete) {
1230         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1231         /* Ok we're finished, but how? */
1232
1233         const int64_t expectedBodySize =
1234             http->storeEntry()->getReply()->bodySize(http->request->method);
1235         if (!http->request->flags.proxyKeepalive && expectedBodySize < 0) {
1236             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
1237             return STREAM_FAILED;
1238         }
1239
1240         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1241             debugs(88, 5, "clientReplyStatus: truncated response body");
1242             return STREAM_UNPLANNED_COMPLETE;
1243         }
1244
1245         if (!done) {
1246             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1247             return STREAM_FAILED;
1248         }
1249
1250         if (expectedBodySize >= 0 && !http->gotEnough()) {
1251             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1252             return STREAM_UNPLANNED_COMPLETE;
1253         }
1254
1255         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1256                http->request->flags.proxyKeepalive);
1257         return STREAM_COMPLETE;
1258     }
1259
1260     // XXX: Should this be checked earlier? We could return above w/o checking.
1261     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1262         /* 4096 is a margin for the HTTP headers included in out.offset */
1263         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1264         return STREAM_FAILED;
1265     }
1266
1267     return STREAM_NONE;
1268 }
1269
1270 /* Responses with no body will not have a content-type header,
1271  * which breaks the rep_mime_type acl, which
1272  * coincidentally, is the most common acl for reply access lists.
1273  * A better long term fix for this is to allow acl matchs on the various
1274  * status codes, and then supply a default ruleset that puts these
1275  * codes before any user defines access entries. That way the user
1276  * can choose to block these responses where appropriate, but won't get
1277  * mysterious breakages.
1278  */
1279 bool
1280 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1281 {
1282     bool result;
1283
1284     switch (sline) {
1285
1286     case Http::scContinue:
1287
1288     case Http::scSwitchingProtocols:
1289
1290     case Http::scProcessing:
1291
1292     case Http::scNoContent:
1293
1294     case Http::scNotModified:
1295         result = true;
1296         break;
1297
1298     default:
1299         result = false;
1300     }
1301
1302     return result;
1303 }
1304
1305 /**
1306  * Generate the reply headers sent to client.
1307  *
1308  * Filters out unwanted entries and hop-by-hop from original reply header
1309  * then adds extra entries if we have more info than origin server
1310  * then adds Squid specific entries
1311  */
1312 void
1313 clientReplyContext::buildReplyHeader()
1314 {
1315     HttpHeader *hdr = &reply->header;
1316     const bool is_hit = http->logType.isTcpHit();
1317     HttpRequest *request = http->request;
1318 #if DONT_FILTER_THESE
1319     /* but you might want to if you run Squid as an HTTP accelerator */
1320     /* hdr->delById(HDR_ACCEPT_RANGES); */
1321     hdr->delById(HDR_ETAG);
1322 #endif
1323
1324     if (is_hit)
1325         hdr->delById(Http::HdrType::SET_COOKIE);
1326     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1327
1328     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1329     // remove the Proxy-Authenticate header
1330     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0))
1331         reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1332
1333     reply->header.removeHopByHopEntries();
1334
1335     //    if (request->range)
1336     //      clientBuildRangeHeader(http, reply);
1337
1338     /*
1339      * Add a estimated Age header on cache hits.
1340      */
1341     if (is_hit) {
1342         /*
1343          * Remove any existing Age header sent by upstream caches
1344          * (note that the existing header is passed along unmodified
1345          * on cache misses)
1346          */
1347         hdr->delById(Http::HdrType::AGE);
1348         /*
1349          * This adds the calculated object age. Note that the details of the
1350          * age calculation is performed by adjusting the timestamp in
1351          * StoreEntry::timestampsSet(), not here.
1352          */
1353         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1354             hdr->delById(Http::HdrType::DATE);
1355             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1356         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1357             // Swap the Date: header to current time if we are simulating an origin
1358             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1359             if (h)
1360                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1361             hdr->delById(Http::HdrType::DATE);
1362             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1363             h = hdr->findEntry(Http::HdrType::EXPIRES);
1364             if (h && http->storeEntry()->expires >= 0) {
1365                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1366                 hdr->delById(Http::HdrType::EXPIRES);
1367                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1368             }
1369             if (http->storeEntry()->timestamp <= squid_curtime) {
1370                 // put X-Cache-Age: instead of Age:
1371                 char age[64];
1372                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1373                 hdr->putExt("X-Cache-Age", age);
1374             }
1375         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1376             hdr->putInt(Http::HdrType::AGE,
1377                         squid_curtime - http->storeEntry()->timestamp);
1378             /* Signal old objects.  NB: rfc 2616 is not clear,
1379              * by implication, on whether we should do this to all
1380              * responses, or only cache hits.
1381              * 14.46 states it ONLY applys for heuristically caclulated
1382              * freshness values, 13.2.4 doesn't specify the same limitation.
1383              * We interpret RFC 2616 under the combination.
1384              */
1385             /* TODO: if maxage or s-maxage is present, don't do this */
1386
1387             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1388                 char tbuf[512];
1389                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1390                           "113", ThisCache,
1391                           "This cache hit is still fresh and more than 1 day old");
1392                 hdr->putStr(Http::HdrType::WARNING, tbuf);
1393             }
1394         }
1395     }
1396
1397     /* RFC 2616: Section 14.18
1398      *
1399      * Add a Date: header if missing.
1400      * We have access to a clock therefore are required to amend any shortcoming in servers.
1401      *
1402      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1403      */
1404     if ( !hdr->has(Http::HdrType::DATE) ) {
1405         if (!http->storeEntry())
1406             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1407         else if (http->storeEntry()->timestamp > 0)
1408             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1409         else {
1410             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1411             /* dump something useful about the problem */
1412             http->storeEntry()->dump(DBG_IMPORTANT);
1413         }
1414     }
1415
1416     // add Warnings required by RFC 2616 if serving a stale hit
1417     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1418         hdr->putWarning(110, "Response is stale");
1419         if (http->request->flags.needValidation)
1420             hdr->putWarning(111, "Revalidation failed");
1421     }
1422
1423     /* Filter unproxyable authentication types */
1424     if (http->logType.oldType != LOG_TCP_DENIED &&
1425             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1426         HttpHeaderPos pos = HttpHeaderInitPos;
1427         HttpHeaderEntry *e;
1428
1429         int connection_auth_blocked = 0;
1430         while ((e = hdr->getEntry(&pos))) {
1431             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1432                 const char *value = e->value.rawBuf();
1433
1434                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1435                         (value[4] == '\0' || value[4] == ' '))
1436                         ||
1437                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1438                          (value[9] == '\0' || value[9] == ' '))
1439                         ||
1440                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1441                          (value[8] == '\0' || value[8] == ' '))) {
1442                     if (request->flags.connectionAuthDisabled) {
1443                         hdr->delAt(pos, connection_auth_blocked);
1444                         continue;
1445                     }
1446                     request->flags.mustKeepalive = true;
1447                     if (!request->flags.accelerated && !request->flags.intercepted) {
1448                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1449                         /*
1450                           We send "Connection: Proxy-Support" header to mark
1451                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1452                           understand the semantics of this header. The RFC should have included
1453                           this recommendation.
1454                         */
1455                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1456                     }
1457                     break;
1458                 }
1459             }
1460         }
1461
1462         if (connection_auth_blocked)
1463             hdr->refreshMask();
1464     }
1465
1466 #if USE_AUTH
1467     /* Handle authentication headers */
1468     if (http->logType.oldType == LOG_TCP_DENIED &&
1469             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1470               reply->sline.status() == Http::scUnauthorized)
1471        ) {
1472         /* Add authentication header */
1473         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1474          * depends on authenticate behaviour: all schemes to date send no extra
1475          * data on 407/401 responses, and do not check the accel state on 401/407
1476          * responses
1477          */
1478         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1479     } else if (request->auth_user_request != NULL)
1480         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1481 #endif
1482
1483     /* Append X-Cache */
1484     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1485                       is_hit ? "HIT" : "MISS", getMyHostname());
1486
1487 #if USE_CACHE_DIGESTS
1488     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1489     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1490                       lookup_type ? lookup_type : "NONE",
1491                       getMyHostname(), getMyPort());
1492
1493 #endif
1494
1495     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1496                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1497                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1498
1499     /* Check whether we should send keep-alive */
1500     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1501         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1502         request->flags.proxyKeepalive = false;
1503     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1504         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1505         request->flags.proxyKeepalive = false;
1506     } else if (request->flags.proxyKeepalive && shutting_down) {
1507         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1508         request->flags.proxyKeepalive = false;
1509     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1510         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1511         request->flags.proxyKeepalive = false;
1512     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1513         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1514         request->flags.proxyKeepalive = false;
1515     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1516         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1517         request->flags.proxyKeepalive = false;
1518     } else if (request->flags.sslBumped && !reply->persistent()) {
1519         // We do not really have to close, but we pretend we are a tunnel.
1520         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1521         request->flags.proxyKeepalive = false;
1522     } else if (request->pinnedConnection() && !reply->persistent()) {
1523         // The peer wants to close the pinned connection
1524         debugs(88, 3, "pinned reply forces close");
1525         request->flags.proxyKeepalive = false;
1526     } else if (http->getConn()) {
1527         ConnStateData * conn = http->getConn();
1528         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1529             // The listening port closed because of a reconfigure
1530             debugs(88, 3, "listening port closed");
1531             request->flags.proxyKeepalive = false;
1532         }
1533     }
1534
1535     // Decide if we send chunked reply
1536     if (maySendChunkedReply &&
1537             request->flags.proxyKeepalive &&
1538             reply->bodySize(request->method) < 0) {
1539         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1540         request->flags.chunkedReply = true;
1541         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1542     }
1543
1544     /* Append VIA */
1545     if (Config.onoff.via) {
1546         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1547         String strVia;
1548         hdr->getList(Http::HdrType::VIA, &strVia);
1549         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1550                  reply->sline.version.major,
1551                  reply->sline.version.minor,
1552                  ThisCache);
1553         strListAdd(&strVia, bbuf, ',');
1554         hdr->delById(Http::HdrType::VIA);
1555         hdr->putStr(Http::HdrType::VIA, strVia.termedBuf());
1556     }
1557     /* Signal keep-alive or close explicitly */
1558     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1559
1560 #if ADD_X_REQUEST_URI
1561     /*
1562      * Knowing the URI of the request is useful when debugging persistent
1563      * connections in a client; we cannot guarantee the order of http headers,
1564      * but X-Request-URI is likely to be the very last header to ease use from a
1565      * debugger [hdr->entries.count-1].
1566      */
1567     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1568                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1569
1570 #endif
1571
1572     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1573     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1574         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1575             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1576         }
1577         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1578     }
1579
1580     httpHdrMangleList(hdr, request, ROR_REPLY);
1581 }
1582
1583 void
1584 clientReplyContext::cloneReply()
1585 {
1586     assert(reply == NULL);
1587
1588     reply = http->storeEntry()->getReply()->clone();
1589     HTTPMSGLOCK(reply);
1590
1591     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1592         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1593         reply->sline.version = Http::ProtocolVersion();
1594     }
1595
1596     /* do header conversions */
1597     buildReplyHeader();
1598 }
1599
1600 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1601 /// We cannot just ignore the entry because it may be locking or otherwise
1602 /// holding an associated cache resource of some sort.
1603 void
1604 clientReplyContext::forgetHit()
1605 {
1606     StoreEntry *e = http->storeEntry();
1607     assert(e); // or we are not dealing with a hit
1608     // We probably have not locked the entry earlier, unfortunately. We lock it
1609     // now so that we can unlock two lines later (and trigger cleanup).
1610     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1611     // used so inconsistently that simply adding locking there leads to bugs.
1612     e->lock("clientReplyContext::forgetHit");
1613     http->storeEntry(NULL);
1614     e->unlock("clientReplyContext::forgetHit"); // may delete e
1615 }
1616
1617 void
1618 clientReplyContext::identifyStoreObject()
1619 {
1620     HttpRequest *r = http->request;
1621
1622     if (r->flags.cachable || r->flags.internal) {
1623         lookingforstore = 5;
1624         StoreEntry::getPublicByRequest (this, r);
1625     } else {
1626         identifyFoundObject (NullStoreEntry::getInstance());
1627     }
1628 }
1629
1630 /**
1631  * Check state of the current StoreEntry object.
1632  * to see if we can determine the final status of the request.
1633  */
1634 void
1635 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1636 {
1637     StoreEntry *e = newEntry;
1638     HttpRequest *r = http->request;
1639
1640     /** \li If the entry received isNull() then we ignore it. */
1641     if (e->isNull()) {
1642         http->storeEntry(NULL);
1643     } else {
1644         http->storeEntry(e);
1645     }
1646
1647     e = http->storeEntry();
1648
1649     /* Release IP-cache entries on reload */
1650     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1651       * 'invalidate' the cached IP entries for this request ???
1652       */
1653     if (r->flags.noCache || r->flags.noCacheHack())
1654         ipcacheInvalidateNegative(r->url.host());
1655
1656 #if USE_CACHE_DIGESTS
1657     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1658 #endif
1659
1660     if (NULL == http->storeEntry()) {
1661         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1662         debugs(85, 3, "StoreEntry is NULL -  MISS");
1663         http->logType = LOG_TCP_MISS;
1664         doGetMoreData();
1665         return;
1666     }
1667
1668     if (Config.onoff.offline) {
1669         /** \li If we are running in offline mode set to HIT */
1670         debugs(85, 3, "offline HIT " << *e);
1671         http->logType = LOG_TCP_HIT;
1672         doGetMoreData();
1673         return;
1674     }
1675
1676     if (http->redirect.status) {
1677         /** \li If redirection status is True force this to be a MISS */
1678         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1679         forgetHit();
1680         http->logType = LOG_TCP_REDIRECT;
1681         doGetMoreData();
1682         return;
1683     }
1684
1685     if (!e->validToSend()) {
1686         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1687         forgetHit();
1688         http->logType = LOG_TCP_MISS;
1689         doGetMoreData();
1690         return;
1691     }
1692
1693     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1694         /* \li Special entries are always hits, no matter what the client says */
1695         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1696         http->logType = LOG_TCP_HIT;
1697         doGetMoreData();
1698         return;
1699     }
1700
1701     if (r->flags.noCache) {
1702         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1703         forgetHit();
1704         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1705         doGetMoreData();
1706         return;
1707     }
1708
1709     debugs(85, 3, "default HIT " << *e);
1710     http->logType = LOG_TCP_HIT;
1711     doGetMoreData();
1712 }
1713
1714 /**
1715  * Request more data from the store for the client Stream
1716  * This is *the* entry point to this module.
1717  *
1718  * Preconditions:
1719  *  - This is the head of the list.
1720  *  - There is at least one more node.
1721  *  - Data context is not null
1722  */
1723 void
1724 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1725 {
1726     /* Test preconditions */
1727     assert(aNode != NULL);
1728     assert(cbdataReferenceValid(aNode));
1729     assert(aNode->node.prev == NULL);
1730     assert(aNode->node.next != NULL);
1731     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1732     assert (context);
1733     assert(context->http == http);
1734
1735     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1736
1737     if (!context->ourNode)
1738         context->ourNode = aNode;
1739
1740     /* no cbdatareference, this is only used once, and safely */
1741     if (context->flags.storelogiccomplete) {
1742         StoreIOBuffer tempBuffer;
1743         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1744         tempBuffer.length = next->readBuffer.length;
1745         tempBuffer.data = next->readBuffer.data;
1746
1747         storeClientCopy(context->sc, http->storeEntry(),
1748                         tempBuffer, clientReplyContext::SendMoreData, context);
1749         return;
1750     }
1751
1752     if (context->http->request->method == Http::METHOD_PURGE) {
1753         context->purgeRequest();
1754         return;
1755     }
1756
1757     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1758
1759     if (context->http->request->method == Http::METHOD_TRACE) {
1760         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1761             context->traceReply(aNode);
1762             return;
1763         }
1764
1765         /* continue forwarding, not finished yet. */
1766         http->logType = LOG_TCP_MISS;
1767
1768         context->doGetMoreData();
1769     } else
1770         context->identifyStoreObject();
1771 }
1772
1773 void
1774 clientReplyContext::doGetMoreData()
1775 {
1776     /* We still have to do store logic processing - vary, cache hit etc */
1777     if (http->storeEntry() != NULL) {
1778         /* someone found the object in the cache for us */
1779         StoreIOBuffer localTempBuffer;
1780
1781         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1782
1783         MemObject *mem_obj = http->storeEntry()->makeMemObject();
1784         if (!mem_obj->hasUris()) {
1785             /*
1786              * This if-block exists because we don't want to clobber
1787              * a preexiting mem_obj->method value if the mem_obj
1788              * already exists.  For example, when a HEAD request
1789              * is a cache hit for a GET response, we want to keep
1790              * the method as GET.
1791              */
1792             mem_obj->setUris(storeId(), http->log_uri, http->request->method);
1793             /**
1794              * Here we can see if the object was
1795              * created using URL or alternative StoreID from helper.
1796              */
1797             debugs(88, 3, "storeId: " << http->storeEntry()->mem_obj->storeId());
1798         }
1799
1800         sc = storeClientListAdd(http->storeEntry(), this);
1801 #if USE_DELAY_POOLS
1802         sc->setDelayId(DelayId::DelayClient(http));
1803 #endif
1804
1805         assert(http->logType.oldType == LOG_TCP_HIT);
1806         reqofs = 0;
1807         /* guarantee nothing has been sent yet! */
1808         assert(http->out.size == 0);
1809         assert(http->out.offset == 0);
1810
1811         if (Ip::Qos::TheConfig.isHitTosActive()) {
1812             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1813         }
1814
1815         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1816             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1817         }
1818
1819         localTempBuffer.offset = reqofs;
1820         localTempBuffer.length = getNextNode()->readBuffer.length;
1821         localTempBuffer.data = getNextNode()->readBuffer.data;
1822         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1823     } else {
1824         /* MISS CASE, http->logType is already set! */
1825         processMiss();
1826     }
1827 }
1828
1829 /** The next node has removed itself from the stream. */
1830 void
1831 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1832 {
1833     /** detach from the stream */
1834     clientStreamDetach(node, http);
1835 }
1836
1837 /**
1838  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1839  * such, writes processed message to the message recipient
1840  */
1841 void
1842 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1843 {
1844     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1845     context->sendMoreData (result);
1846 }
1847
1848 void
1849 clientReplyContext::makeThisHead()
1850 {
1851     /* At least, I think thats what this does */
1852     dlinkDelete(&http->active, &ClientActiveRequests);
1853     dlinkAdd(http, &http->active, &ClientActiveRequests);
1854 }
1855
1856 bool
1857 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1858 {
1859     return /* aborted request */
1860         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1861         /* Upstream read error */ (result.flags.error) ||
1862         /* Upstream EOF */ (sizeToProcess == 0);
1863 }
1864
1865 void
1866 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1867 {
1868     /** call clientWriteComplete so the client socket gets closed
1869      *
1870      * We call into the stream, because we don't know that there is a
1871      * client socket!
1872      */
1873     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1874     StoreIOBuffer localTempBuffer;
1875     flags.complete = 1;
1876     http->request->flags.streamError = true;
1877     localTempBuffer.flags.error = result.flags.error;
1878     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1879                          localTempBuffer);
1880 }
1881
1882 void
1883 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1884 {
1885     StoreIOBuffer localTempBuffer;
1886
1887     if (result.length == 0) {
1888         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1889         flags.complete = 1;
1890     }
1891
1892     assert(result.offset - headers_sz == next()->readBuffer.offset);
1893     localTempBuffer.offset = result.offset - headers_sz;
1894     localTempBuffer.length = result.length;
1895
1896     if (localTempBuffer.length)
1897         localTempBuffer.data = source;
1898
1899     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1900                          localTempBuffer);
1901 }
1902
1903 clientStreamNode *
1904 clientReplyContext::next() const
1905 {
1906     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1907     return getNextNode();
1908 }
1909
1910 void
1911 clientReplyContext::sendBodyTooLargeError()
1912 {
1913     Ip::Address tmp_noaddr;
1914     tmp_noaddr.setNoAddr(); // TODO: make a global const
1915     http->logType = LOG_TCP_DENIED_REPLY;
1916     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1917                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1918                                        http->request);
1919     removeClientStoreReference(&(sc), http);
1920     HTTPMSGUNLOCK(reply);
1921     startError(err);
1922
1923 }
1924
1925 /// send 412 (Precondition Failed) to client
1926 void
1927 clientReplyContext::sendPreconditionFailedError()
1928 {
1929     http->logType = LOG_TCP_HIT;
1930     ErrorState *const err =
1931         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1932                          NULL, http->getConn()->clientConnection->remote, http->request);
1933     removeClientStoreReference(&sc, http);
1934     HTTPMSGUNLOCK(reply);
1935     startError(err);
1936 }
1937
1938 /// send 304 (Not Modified) to client
1939 void
1940 clientReplyContext::sendNotModified()
1941 {
1942     StoreEntry *e = http->storeEntry();
1943     const time_t timestamp = e->timestamp;
1944     HttpReply *const temprep = e->getReply()->make304();
1945     http->logType = LOG_TCP_IMS_HIT;
1946     removeClientStoreReference(&sc, http);
1947     createStoreEntry(http->request->method, RequestFlags());
1948     e = http->storeEntry();
1949     // Copy timestamp from the original entry so the 304
1950     // reply has a meaningful Age: header.
1951     e->timestampsSet();
1952     e->timestamp = timestamp;
1953     e->replaceHttpReply(temprep);
1954     e->complete();
1955     /*
1956      * TODO: why put this in the store and then serialise it and
1957      * then parse it again. Simply mark the request complete in
1958      * our context and write the reply struct to the client side.
1959      */
1960     triggerInitialStoreRead();
1961 }
1962
1963 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1964 /// depending on request method
1965 void
1966 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1967 {
1968     if (http->request->method == Http::METHOD_GET ||
1969             http->request->method == Http::METHOD_HEAD)
1970         sendNotModified();
1971     else
1972         sendPreconditionFailedError();
1973 }
1974
1975 void
1976 clientReplyContext::processReplyAccess ()
1977 {
1978     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
1979     assert(reply);
1980
1981     /** Don't block our own responses or HTTP status messages */
1982     if (http->logType.oldType == LOG_TCP_DENIED ||
1983             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
1984             alwaysAllowResponse(reply->sline.status())) {
1985         headers_sz = reply->hdr_sz;
1986         processReplyAccessResult(ACCESS_ALLOWED);
1987         return;
1988     }
1989
1990     /** Check for reply to big error */
1991     if (reply->expectedBodyTooLarge(*http->request)) {
1992         sendBodyTooLargeError();
1993         return;
1994     }
1995
1996     headers_sz = reply->hdr_sz;
1997
1998     /** check for absent access controls (permit by default) */
1999     if (!Config.accessList.reply) {
2000         processReplyAccessResult(ACCESS_ALLOWED);
2001         return;
2002     }
2003
2004     /** Process http_reply_access lists */
2005     ACLFilledChecklist *replyChecklist =
2006         clientAclChecklistCreate(Config.accessList.reply, http);
2007     replyChecklist->reply = reply;
2008     HTTPMSGLOCK(replyChecklist->reply);
2009     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2010 }
2011
2012 void
2013 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2014 {
2015     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2016     me->processReplyAccessResult(rv);
2017 }
2018
2019 void
2020 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2021 {
2022     debugs(88, 2, "The reply for " << http->request->method
2023            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2024            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2025
2026     if (accessAllowed != ACCESS_ALLOWED) {
2027         ErrorState *err;
2028         err_type page_id;
2029         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2030
2031         http->logType = LOG_TCP_DENIED_REPLY;
2032
2033         if (page_id == ERR_NONE)
2034             page_id = ERR_ACCESS_DENIED;
2035
2036         Ip::Address tmp_noaddr;
2037         tmp_noaddr.setNoAddr();
2038         err = clientBuildError(page_id, Http::scForbidden, NULL,
2039                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2040                                http->request);
2041
2042         removeClientStoreReference(&sc, http);
2043
2044         HTTPMSGUNLOCK(reply);
2045
2046         startError(err);
2047
2048         return;
2049     }
2050
2051     /* Ok, the reply is allowed, */
2052     http->loggingEntry(http->storeEntry());
2053
2054     ssize_t body_size = reqofs - reply->hdr_sz;
2055     if (body_size < 0) {
2056         reqofs = reply->hdr_sz;
2057         body_size = 0;
2058     }
2059
2060     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2061            (int) body_size << " bytes after " << reply->hdr_sz <<
2062            " bytes of headers");
2063
2064 #if USE_SQUID_ESI
2065
2066     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2067             !alwaysAllowResponse(reply->sline.status()) &&
2068             esiEnableProcessing(reply)) {
2069         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2070         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2071                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2072     }
2073
2074 #endif
2075
2076     if (http->request->method == Http::METHOD_HEAD) {
2077         /* do not forward body for HEAD replies */
2078         body_size = 0;
2079         http->flags.done_copying = true;
2080         flags.complete = 1;
2081     }
2082
2083     assert (!flags.headersSent);
2084     flags.headersSent = true;
2085
2086     StoreIOBuffer localTempBuffer;
2087     char *buf = next()->readBuffer.data;
2088     char *body_buf = buf + reply->hdr_sz;
2089
2090     //Server side may disable ranges under some circumstances.
2091
2092     if ((!http->request->range))
2093         next()->readBuffer.offset = 0;
2094
2095     body_buf -= next()->readBuffer.offset;
2096
2097     if (next()->readBuffer.offset != 0) {
2098         if (next()->readBuffer.offset > body_size) {
2099             /* Can't use any of the body we received. send nothing */
2100             localTempBuffer.length = 0;
2101             localTempBuffer.data = NULL;
2102         } else {
2103             localTempBuffer.length = body_size - next()->readBuffer.offset;
2104             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2105         }
2106     } else {
2107         localTempBuffer.length = body_size;
2108         localTempBuffer.data = body_buf;
2109     }
2110
2111     /* TODO??: move the data in the buffer back by the request header size */
2112     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2113                          http, reply, localTempBuffer);
2114
2115     return;
2116 }
2117
2118 void
2119 clientReplyContext::sendMoreData (StoreIOBuffer result)
2120 {
2121     if (deleting)
2122         return;
2123
2124     StoreEntry *entry = http->storeEntry();
2125
2126     ConnStateData * conn = http->getConn();
2127
2128     // too late, our conn is closing
2129     // TODO: should we also quit?
2130     if (conn == NULL) {
2131         debugs(33,3, "not sending more data to a closed connection" );
2132         return;
2133     }
2134     if (!conn->isOpen()) {
2135         debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2136         return;
2137     }
2138     if (conn->pinning.zeroReply) {
2139         debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2140         return;
2141     }
2142
2143     char *buf = next()->readBuffer.data;
2144
2145     if (buf != result.data) {
2146         /* we've got to copy some data */
2147         assert(result.length <= next()->readBuffer.length);
2148         memcpy(buf, result.data, result.length);
2149     }
2150
2151     if (reqofs==0 && !http->logType.isTcpHit() && Comm::IsConnOpen(conn->clientConnection)) {
2152         if (Ip::Qos::TheConfig.isHitTosActive()) {
2153             Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2154         }
2155         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2156             Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2157         }
2158     }
2159
2160     /* We've got the final data to start pushing... */
2161     flags.storelogiccomplete = 1;
2162
2163     reqofs += result.length;
2164
2165     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2166
2167     assert(http->request != NULL);
2168
2169     /* ESI TODO: remove this assert once everything is stable */
2170     assert(http->client_stream.head->data
2171            && cbdataReferenceValid(http->client_stream.head->data));
2172
2173     makeThisHead();
2174
2175     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2176            reqofs << " bytes (" << result.length <<
2177            " new bytes)");
2178     debugs(88, 5, "clientReplyContext::sendMoreData:"
2179            << conn->clientConnection <<
2180            " '" << entry->url() << "'" <<
2181            " out.offset=" << http->out.offset);
2182
2183     /* update size of the request */
2184     reqsize = reqofs;
2185
2186     if (errorInStream(result, reqofs)) {
2187         sendStreamError(result);
2188         return;
2189     }
2190
2191     if (flags.headersSent) {
2192         pushStreamData (result, buf);
2193         return;
2194     }
2195
2196     cloneReply();
2197
2198 #if USE_DELAY_POOLS
2199     if (sc)
2200         sc->setDelayId(DelayId::DelayClient(http,reply));
2201 #endif
2202
2203     /* handle headers */
2204
2205     if (Config.onoff.log_mime_hdrs) {
2206         size_t k;
2207
2208         if ((k = headersEnd(buf, reqofs))) {
2209             safe_free(http->al->headers.reply);
2210             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2211             xstrncpy(http->al->headers.reply, buf, k);
2212         }
2213     }
2214
2215     holdingBuffer = result;
2216     processReplyAccess();
2217     return;
2218 }
2219
2220 /* Using this breaks the client layering just a little!
2221  */
2222 void
2223 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2224 {
2225     assert(http != NULL);
2226     /*
2227      * For erroneous requests, we might not have a h->request,
2228      * so make a fake one.
2229      */
2230
2231     if (http->request == NULL) {
2232         http->request = new HttpRequest(m, AnyP::PROTO_NONE, null_string);
2233         HTTPMSGLOCK(http->request);
2234     }
2235
2236     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2237
2238     // Make entry collapsable ASAP, to increase collapsing chances for others,
2239     // TODO: every must-revalidate and similar request MUST reach the origin,
2240     // but do we have to prohibit others from collapsing on that request?
2241     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2242             !reqFlags.needValidation &&
2243             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2244         // make the entry available for future requests now
2245         Store::Root().allowCollapsing(e, reqFlags, m);
2246     }
2247
2248     sc = storeClientListAdd(e, this);
2249
2250 #if USE_DELAY_POOLS
2251     sc->setDelayId(DelayId::DelayClient(http));
2252 #endif
2253
2254     reqofs = 0;
2255
2256     reqsize = 0;
2257
2258     /* I don't think this is actually needed! -- adrian */
2259     /* http->reqbuf = http->norm_reqbuf; */
2260     //    assert(http->reqbuf == http->norm_reqbuf);
2261     /* The next line is illegal because we don't know if the client stream
2262      * buffers have been set up
2263      */
2264     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2265     //        SendMoreData, this);
2266     /* So, we mark the store logic as complete */
2267     flags.storelogiccomplete = 1;
2268
2269     /* and get the caller to request a read, from whereever they are */
2270     /* NOTE: after ANY data flows down the pipe, even one step,
2271      * this function CAN NOT be used to manage errors
2272      */
2273     http->storeEntry(e);
2274 }
2275
2276 ErrorState *
2277 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2278                  Ip::Address &src_addr, HttpRequest * request)
2279 {
2280     ErrorState *err = new ErrorState(page_id, status, request);
2281     err->src_addr = src_addr;
2282
2283     if (url)
2284         err->url = xstrdup(url);
2285
2286     return err;
2287 }
2288