src/client_side_reply.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 88    Client-side Reply Routines
   5  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34 #include "config.h"
  35
  36 /* for ClientActiveRequests global */
  37 #include "dlink.h"
  38
  39 /* old includes without reasons given. */
  40 #include "squid.h"
  41 #include "acl/FilledChecklist.h"
  42 #include "acl/Gadgets.h"
  43 #if USE_AUTH
  44 #include "auth/UserRequest.h"
  45 #endif
  46 #include "client_side.h"
  47 #include "client_side_reply.h"
  48 #include "clientStream.h"
  49 #if USE_DELAY_POOLS
  50 #include "DelayPools.h"
  51 #endif
  52 #include "errorpage.h"
  53 #if USE_SQUID_ESI
  54 #include "esi/Esi.h"
  55 #endif
  56 #include "fde.h"
  57 #include "forward.h"
  58 #include "format/Tokens.h"
  59 #include "HttpReply.h"
  60 #include "HttpRequest.h"
  61 #include "ip/QosConfig.h"
  62 #include "ipcache.h"
  63 #include "MemObject.h"
  64 #include "ProtoPort.h"
  65 #include "SquidTime.h"
  66 #include "StoreClient.h"
  67 #include "Store.h"
  68
  69 CBDATA_CLASS_INIT(clientReplyContext);
  70
  71 /* Local functions */
  72 extern "C" CSS clientReplyStatus;
  73 extern ErrorState *clientBuildError(err_type, http_status, char const *, Ip::Address &, HttpRequest *);
  74
  75 /* privates */
  76
  77 clientReplyContext::~clientReplyContext()
  78 {
  79     deleting = true;
  80     /* This may trigger a callback back into SendMoreData as the cbdata
  81      * is still valid
  82      */
  83     removeClientStoreReference(&sc, http);
  84     /* old_entry might still be set if we didn't yet get the reply
  85      * code in HandleIMSReply() */
  86     removeStoreReference(&old_sc, &old_entry);
  87     safe_free(tempBuffer.data);
  88     cbdataReferenceDone(http);
  89     HTTPMSGUNLOCK(reply);
  90 }
  91
  92 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) : http (cbdataReference(clientContext)), old_entry (NULL), old_sc(NULL), deleting(false)
  93 {}
  94
  95 /** Create an error in the store awaiting the client side to read it.
  96  *
  97  * This may be better placed in the clientStream logic, but it has not been
  98  * relocated there yet
  99  */
 100 void
 101 clientReplyContext::setReplyToError(
 102     err_type err, http_status status, const HttpRequestMethod& method, char const *uri,
 103     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 104 #if USE_AUTH
 105     AuthUserRequest::Pointer auth_user_request
 106 #else
 107     void*
 108 #endif
 109 )
 110 {
 111     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 112
 113     if (unparsedrequest)
 114         errstate->request_hdrs = xstrdup(unparsedrequest);
 115
 116     if (status == HTTP_NOT_IMPLEMENTED && http->request)
 117         /* prevent confusion over whether we default to persistent or not */
 118         http->request->flags.proxy_keepalive = 0;
 119
 120     http->al.http.code = errstate->httpStatus;
 121
 122     createStoreEntry(method, request_flags());
 123 #if USE_AUTH
 124     errstate->auth_user_request = auth_user_request;
 125 #endif
 126     assert(errstate->callback_data == NULL);
 127     errorAppendEntry(http->storeEntry(), errstate);
 128     /* Now the caller reads to get this */
 129 }
 130
 131 void
 132 clientReplyContext::removeStoreReference(store_client ** scp,
 133         StoreEntry ** ep)
 134 {
 135     StoreEntry *e;
 136     store_client *sc_tmp = *scp;
 137
 138     if ((e = *ep) != NULL) {
 139         *ep = NULL;
 140         storeUnregister(sc_tmp, e, this);
 141         *scp = NULL;
 142         e->unlock();
 143     }
 144 }
 145
 146 void
 147 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 148 {
 149     StoreEntry *reference = aHttpRequest->storeEntry();
 150     removeStoreReference(scp, &reference);
 151     aHttpRequest->storeEntry(reference);
 152 }
 153
 154 void *
 155 clientReplyContext::operator new (size_t byteCount)
 156 {
 157     /* derived classes with different sizes must implement their own new */
 158     assert (byteCount == sizeof (clientReplyContext));
 159     CBDATA_INIT_TYPE(clientReplyContext);
 160     return cbdataAlloc(clientReplyContext);
 161 }
 162
 163 void
 164 clientReplyContext::operator delete (void *address)
 165 {
 166     clientReplyContext * tmp = (clientReplyContext *)address;
 167     cbdataFree (tmp);
 168 }
 169
 170 void
 171 clientReplyContext::saveState()
 172 {
 173     assert(old_sc == NULL);
 174     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 175     old_entry = http->storeEntry();
 176     old_sc = sc;
 177     old_reqsize = reqsize;
 178     tempBuffer.offset = reqofs;
 179     /* Prevent accessing the now saved entries */
 180     http->storeEntry(NULL);
 181     sc = NULL;
 182     reqsize = 0;
 183     reqofs = 0;
 184 }
 185
 186 void
 187 clientReplyContext::restoreState()
 188 {
 189     assert(old_sc != NULL);
 190     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 191     removeClientStoreReference(&sc, http);
 192     http->storeEntry(old_entry);
 193     sc = old_sc;
 194     reqsize = old_reqsize;
 195     reqofs = tempBuffer.offset;
 196     /* Prevent accessed the old saved entries */
 197     old_entry = NULL;
 198     old_sc = NULL;
 199     old_reqsize = 0;
 200     tempBuffer.offset = 0;
 201 }
 202
 203 void
 204 clientReplyContext::startError(ErrorState * err)
 205 {
 206     createStoreEntry(http->request->method, request_flags());
 207     triggerInitialStoreRead();
 208     errorAppendEntry(http->storeEntry(), err);
 209 }
 210
 211 clientStreamNode *
 212 clientReplyContext::getNextNode() const
 213 {
 214     return (clientStreamNode *)ourNode->node.next->data;
 215 }
 216
 217 /* This function is wrong - the client parameters don't include the
 218  * header offset
 219  */
 220 void
 221 clientReplyContext::triggerInitialStoreRead()
 222 {
 223     /* when confident, 0 becomes reqofs, and then this factors into
 224      * startSendProcess
 225      */
 226     assert(reqofs == 0);
 227     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 228     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 229 }
 230
 231 /* there is an expired entry in the store.
 232  * setup a temporary buffer area and perform an IMS to the origin
 233  */
 234 void
 235 clientReplyContext::processExpired()
 236 {
 237     char *url = http->uri;
 238     StoreEntry *entry = NULL;
 239     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 240     assert(http->storeEntry()->lastmod >= 0);
 241     /*
 242      * check if we are allowed to contact other servers
 243      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 244      *      a stale entry *if* it matches client requirements
 245      */
 246
 247     if (http->onlyIfCached()) {
 248         processOnlyIfCachedMiss();
 249         return;
 250     }
 251
 252     http->request->flags.refresh = 1;
 253 #if STORE_CLIENT_LIST_DEBUG
 254     /* Prevent a race with the store client memory free routines
 255      */
 256     assert(storeClientIsThisAClient(sc, this));
 257 #endif
 258     /* Prepare to make a new temporary request */
 259     saveState();
 260     entry = storeCreateEntry(url,
 261                              http->log_uri, http->request->flags, http->request->method);
 262     /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 263     sc = storeClientListAdd(entry, this);
 264 #if USE_DELAY_POOLS
 265     /* delay_id is already set on original store client */
 266     sc->setDelayId(DelayId::DelayClient(http));
 267 #endif
 268
 269     http->request->lastmod = old_entry->lastmod;
 270     debugs(88, 5, "clientReplyContext::processExpired : lastmod " << entry->lastmod );
 271     http->storeEntry(entry);
 272     assert(http->out.offset == 0);
 273     assert(http->request->clientConnectionManager == http->getConn());
 274
 275     /*
 276      * A refcounted pointer so that FwdState stays around as long as
 277      * this clientReplyContext does
 278      */
 279     Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 280     FwdState::fwdStart(conn, http->storeEntry(), http->request);
 281
 282     /* Register with storage manager to receive updates when data comes in. */
 283
 284     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 285         debugs(88, 0, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 286
 287     {
 288         /* start counting the length from 0 */
 289         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 290         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 291     }
 292 }
 293
 294
 295 void
 296 clientReplyContext::sendClientUpstreamResponse()
 297 {
 298     StoreIOBuffer tempresult;
 299     removeStoreReference(&old_sc, &old_entry);
 300     /* here the data to send is the data we just received */
 301     tempBuffer.offset = 0;
 302     old_reqsize = 0;
 303     /* sendMoreData tracks the offset as well.
 304      * Force it back to zero */
 305     reqofs = 0;
 306     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 307     /* TODO: provide sendMoreData with the ready parsed reply */
 308     tempresult.length = reqsize;
 309     tempresult.data = tempbuf;
 310     sendMoreData(tempresult);
 311 }
 312
 313 void
 314 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 315 {
 316     clientReplyContext *context = (clientReplyContext *)data;
 317     context->handleIMSReply(result);
 318 }
 319
 320 void
 321 clientReplyContext::sendClientOldEntry()
 322 {
 323     /* Get the old request back */
 324     restoreState();
 325     /* here the data to send is in the next nodes buffers already */
 326     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 327     /* sendMoreData tracks the offset as well.
 328      * Force it back to zero */
 329     reqofs = 0;
 330     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 331     sendMoreData(tempresult);
 332 }
 333
 334 /* This is the workhorse of the HandleIMSReply callback.
 335  *
 336  * It is called when we've got data back from the origin following our
 337  * IMS request to revalidate a stale entry.
 338  */
 339 void
 340 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 341 {
 342     if (deleting)
 343         return;
 344
 345     debugs(88, 3, "handleIMSReply: " << http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes" );
 346
 347     if (http->storeEntry() == NULL)
 348         return;
 349
 350     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 351         return;
 352
 353     /* update size of the request */
 354     reqsize = result.length + reqofs;
 355
 356     const http_status status = http->storeEntry()->getReply()->sline.status;
 357
 358     // request to origin was aborted
 359     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 360         debugs(88, 3, "handleIMSReply: request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client" );
 361         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 362         sendClientOldEntry();
 363     }
 364
 365     HttpReply *old_rep = (HttpReply *) old_entry->getReply();
 366
 367     // origin replied 304
 368     if (status == HTTP_NOT_MODIFIED) {
 369         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 370
 371         // update headers on existing entry
 372         old_rep->updateOnNotModified(http->storeEntry()->getReply());
 373         old_entry->timestampsSet();
 374
 375         // if client sent IMS
 376
 377         if (http->request->flags.ims && !old_entry->modifiedSince(http->request)) {
 378             // forward the 304 from origin
 379             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and forwarding 304 to client");
 380             sendClientUpstreamResponse();
 381         } else {
 382             // send existing entry, it's still valid
 383             debugs(88, 3, "handleIMSReply: origin replied 304, revalidating existing entry and sending " <<
 384                    old_rep->sline.status << " to client");
 385             sendClientOldEntry();
 386         }
 387     }
 388
 389     // origin replied with a non-error code
 390     else if (status > HTTP_STATUS_NONE && status < HTTP_INTERNAL_SERVER_ERROR) {
 391         // forward response from origin
 392         http->logType = LOG_TCP_REFRESH_MODIFIED;
 393         debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client");
 394         sendClientUpstreamResponse();
 395     }
 396
 397     // origin replied with an error
 398     else if (http->request->flags.fail_on_validation_err) {
 399         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 400         debugs(88, 3, "handleIMSReply: origin replied with error " << status <<
 401                ", forwarding to client due to fail_on_validation_err");
 402         sendClientUpstreamResponse();
 403     } else {
 404         // ignore and let client have old entry
 405         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 406         debugs(88, 3, "handleIMSReply: origin replied with error " <<
 407                status << ", sending old entry (" << old_rep->sline.status << ") to client");
 408         sendClientOldEntry();
 409     }
 410 }
 411
 412 extern "C" CSR clientGetMoreData;
 413 extern "C" CSD clientReplyDetach;
 414
 415 /**
 416  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 417  * have been parsed.  Normally this should be a single call, but
 418  * it might take more than one.  As soon as we have the headers,
 419  * we hand off to clientSendMoreData, processExpired, or
 420  * processMiss.
 421  */
 422 void
 423 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 424 {
 425     clientReplyContext *context = (clientReplyContext *)data;
 426     context->cacheHit(result);
 427 }
 428
 429 /**
 430  * Process a possible cache HIT.
 431  */
 432 void
 433 clientReplyContext::cacheHit(StoreIOBuffer result)
 434 {
 435     /** Ignore if the HIT object is being deleted. */
 436     if (deleting)
 437         return;
 438
 439     StoreEntry *e = http->storeEntry();
 440
 441     HttpRequest *r = http->request;
 442
 443     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 444
 445     if (http->storeEntry() == NULL) {
 446         debugs(88, 3, "clientCacheHit: request aborted");
 447         return;
 448     } else if (result.flags.error) {
 449         /* swap in failure */
 450         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 451         http->logType = LOG_TCP_SWAPFAIL_MISS;
 452         removeClientStoreReference(&sc, http);
 453         processMiss();
 454         return;
 455     }
 456
 457     if (result.length == 0) {
 458         /* the store couldn't get enough data from the file for us to id the
 459          * object
 460          */
 461         /* treat as a miss */
 462         http->logType = LOG_TCP_MISS;
 463         processMiss();
 464         return;
 465     }
 466
 467     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 468     /* update size of the request */
 469     reqsize = result.length + reqofs;
 470
 471     /*
 472      * Got the headers, now grok them
 473      */
 474     assert(http->logType == LOG_TCP_HIT);
 475
 476     if (strcmp(e->mem_obj->url, urlCanonical(r)) != 0) {
 477         debugs(33, 1, "clientProcessHit: URL mismatch, '" << e->mem_obj->url << "' != '" << urlCanonical(r) << "'");
 478         processMiss();
 479         return;
 480     }
 481
 482     switch (varyEvaluateMatch(e, r)) {
 483
 484     case VARY_NONE:
 485         /* No variance detected. Continue as normal */
 486         break;
 487
 488     case VARY_MATCH:
 489         /* This is the correct entity for this request. Continue */
 490         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 491         break;
 492
 493     case VARY_OTHER:
 494         /* This is not the correct entity for this request. We need
 495          * to requery the cache.
 496          */
 497         removeClientStoreReference(&sc, http);
 498         e = NULL;
 499         /* Note: varyEvalyateMatch updates the request with vary information
 500          * so we only get here once. (it also takes care of cancelling loops)
 501          */
 502         debugs(88, 2, "clientProcessHit: Vary detected!");
 503         clientGetMoreData(ourNode, http);
 504         return;
 505
 506     case VARY_CANCEL:
 507         /* varyEvaluateMatch found a object loop. Process as miss */
 508         debugs(88, 1, "clientProcessHit: Vary object loop!");
 509         processMiss();
 510         return;
 511     }
 512
 513     if (r->method == METHOD_PURGE) {
 514         removeClientStoreReference(&sc, http);
 515         e = NULL;
 516         purgeRequest();
 517         return;
 518     }
 519
 520     if (e->checkNegativeHit()
 521 #if USE_HTTP_VIOLATIONS
 522             && !r->flags.nocache_hack
 523 #endif
 524        ) {
 525         http->logType = LOG_TCP_NEGATIVE_HIT;
 526         sendMoreData(result);
 527     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 528         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 529         /*
 530          * We hold a stale copy; it needs to be validated
 531          */
 532         /*
 533          * The 'need_validation' flag is used to prevent forwarding
 534          * loops between siblings.  If our copy of the object is stale,
 535          * then we should probably only use parents for the validation
 536          * request.  Otherwise two siblings could generate a loop if
 537          * both have a stale version of the object.
 538          */
 539         r->flags.need_validation = 1;
 540
 541         if (e->lastmod < 0) {
 542             /*
 543              * Previous reply didn't have a Last-Modified header,
 544              * we cannot revalidate it.
 545              */
 546             http->logType = LOG_TCP_MISS;
 547             processMiss();
 548         } else if (r->flags.nocache) {
 549             /*
 550              * This did not match a refresh pattern that overrides no-cache
 551              * we should honour the client no-cache header.
 552              */
 553             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 554             processMiss();
 555         } else if (r->protocol == AnyP::PROTO_HTTP) {
 556             /*
 557              * Object needs to be revalidated
 558              * XXX This could apply to FTP as well, if Last-Modified is known.
 559              */
 560             processExpired();
 561         } else {
 562             /*
 563              * We don't know how to re-validate other protocols. Handle
 564              * them as if the object has expired.
 565              */
 566             http->logType = LOG_TCP_MISS;
 567             processMiss();
 568         }
 569     } else if (r->conditional())
 570         processConditional(result);
 571     else {
 572         /*
 573          * plain ol' cache hit
 574          */
 575
 576 #if USE_DELAY_POOLS
 577         if (e->store_status != STORE_OK)
 578             http->logType = LOG_TCP_MISS;
 579         else
 580 #endif
 581             if (e->mem_status == IN_MEMORY)
 582                 http->logType = LOG_TCP_MEM_HIT;
 583             else if (Config.onoff.offline)
 584                 http->logType = LOG_TCP_OFFLINE_HIT;
 585
 586         sendMoreData(result);
 587     }
 588 }
 589
 590 /**
 591  * Prepare to fetch the object as it's a cache miss of some kind.
 592  */
 593 void
 594 clientReplyContext::processMiss()
 595 {
 596     char *url = http->uri;
 597     HttpRequest *r = http->request;
 598     ErrorState *err = NULL;
 599     debugs(88, 4, "clientProcessMiss: '" << RequestMethodStr(r->method) << " " << url << "'");
 600
 601     /**
 602      * We might have a left-over StoreEntry from a failed cache hit
 603      * or IMS request.
 604      */
 605     if (http->storeEntry()) {
 606         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 607             debugs(88, 0, "clientProcessMiss: miss on a special object (" << url << ").");
 608             debugs(88, 0, "\tlog_type = " << Format::log_tags[http->logType]);
 609             http->storeEntry()->dump(1);
 610         }
 611
 612         removeClientStoreReference(&sc, http);
 613     }
 614
 615     /** Check if its a PURGE request to be actioned. */
 616     if (r->method == METHOD_PURGE) {
 617         purgeRequest();
 618         return;
 619     }
 620
 621     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 622     if (r->method == METHOD_OTHER) {
 623         purgeAllCached();
 624     }
 625
 626     /** Check if 'only-if-cached' flag is set. Action if so. */
 627     if (http->onlyIfCached()) {
 628         processOnlyIfCachedMiss();
 629         return;
 630     }
 631
 632     /// Deny loops for accelerator and interceptor. TODO: deny in all modes?
 633     if (r->flags.loopdetect &&
 634             (http->flags.accel || http->flags.intercepted)) {
 635         http->al.http.code = HTTP_FORBIDDEN;
 636         err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, http->getConn()->clientConnection->remote, http->request);
 637         createStoreEntry(r->method, request_flags());
 638         errorAppendEntry(http->storeEntry(), err);
 639         triggerInitialStoreRead();
 640         return;
 641     } else {
 642         assert(http->out.offset == 0);
 643         createStoreEntry(r->method, r->flags);
 644         triggerInitialStoreRead();
 645
 646         if (http->redirect.status) {
 647             HttpReply *rep = new HttpReply;
 648 #if LOG_TCP_REDIRECTS
 649
 650             http->logType = LOG_TCP_REDIRECT;
 651 #endif
 652
 653             http->storeEntry()->releaseRequest();
 654             rep->redirect(http->redirect.status, http->redirect.location);
 655             http->storeEntry()->replaceHttpReply(rep);
 656             http->storeEntry()->complete();
 657             return;
 658         }
 659
 660         /** Check for internal requests. Update Protocol info if so. */
 661         if (http->flags.internal)
 662             r->protocol = AnyP::PROTO_INTERNAL;
 663
 664         assert(r->clientConnectionManager == http->getConn());
 665
 666         /** Start forwarding to get the new object from network */
 667         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 668         FwdState::fwdStart(conn, http->storeEntry(), r);
 669     }
 670 }
 671
 672 /**
 673  * client issued a request with an only-if-cached cache-control directive;
 674  * we did not find a cached object that can be returned without
 675  *     contacting other servers;
 676  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 677  */
 678 void
 679 clientReplyContext::processOnlyIfCachedMiss()
 680 {
 681     debugs(88, 4, "clientProcessOnlyIfCachedMiss: '" <<
 682            RequestMethodStr(http->request->method) << " " << http->uri << "'");
 683     http->al.http.code = HTTP_GATEWAY_TIMEOUT;
 684     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, HTTP_GATEWAY_TIMEOUT, NULL,
 685                                        http->getConn()->clientConnection->remote, http->request);
 686     removeClientStoreReference(&sc, http);
 687     startError(err);
 688 }
 689
 690 /// process conditional request from client
 691 void
 692 clientReplyContext::processConditional(StoreIOBuffer &result)
 693 {
 694     StoreEntry *const e = http->storeEntry();
 695
 696     if (e->getReply()->sline.status != HTTP_OK) {
 697         debugs(88, 4, "clientReplyContext::processConditional: Reply code " <<
 698                e->getReply()->sline.status << " != 200");
 699         http->logType = LOG_TCP_MISS;
 700         processMiss();
 701         return;
 702     }
 703
 704     HttpRequest &r = *http->request;
 705
 706     if (r.header.has(HDR_IF_MATCH) && !e->hasIfMatchEtag(r)) {
 707         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 708         sendPreconditionFailedError();
 709         return;
 710     }
 711
 712     bool matchedIfNoneMatch = false;
 713     if (r.header.has(HDR_IF_NONE_MATCH)) {
 714         if (!e->hasIfNoneMatchEtag(r)) {
 715             // RFC 2616: ignore IMS if If-None-Match did not match
 716             r.flags.ims = 0;
 717             r.ims = -1;
 718             r.imslen = 0;
 719             r.header.delById(HDR_IF_MODIFIED_SINCE);
 720             http->logType = LOG_TCP_MISS;
 721             sendMoreData(result);
 722             return;
 723         }
 724
 725         if (!r.flags.ims) {
 726             // RFC 2616: if If-None-Match matched and there is no IMS,
 727             // reply with 304 Not Modified or 412 Precondition Failed
 728             sendNotModifiedOrPreconditionFailedError();
 729             return;
 730         }
 731
 732         // otherwise check IMS below to decide if we reply with 304 or 412
 733         matchedIfNoneMatch = true;
 734     }
 735
 736     if (r.flags.ims) {
 737         // handle If-Modified-Since requests from the client
 738         if (e->modifiedSince(&r)) {
 739             http->logType = LOG_TCP_IMS_HIT;
 740             sendMoreData(result);
 741             return;
 742         }
 743
 744         if (matchedIfNoneMatch) {
 745             // If-None-Match matched, reply with 304 Not Modified or
 746             // 412 Precondition Failed
 747             sendNotModifiedOrPreconditionFailedError();
 748             return;
 749         }
 750
 751         // otherwise reply with 304 Not Modified
 752         sendNotModified();
 753     }
 754 }
 755
 756 void
 757 clientReplyContext::purgeRequestFindObjectToPurge()
 758 {
 759     /* Try to find a base entry */
 760     http->flags.purging = 1;
 761     lookingforstore = 1;
 762
 763     // TODO: can we use purgeAllCached() here instead of doing the
 764     // getPublicByRequestMethod() dance?
 765     StoreEntry::getPublicByRequestMethod(this, http->request, METHOD_GET);
 766 }
 767
 768 // Purges all entries with a given url
 769 // TODO: move to SideAgent parent, when we have one
 770 /*
 771  * We probably cannot purge Vary-affected responses because their MD5
 772  * keys depend on vary headers.
 773  */
 774 void
 775 purgeEntriesByUrl(HttpRequest * req, const char *url)
 776 {
 777 #if USE_HTCP
 778     bool get_or_head_sent = false;
 779 #endif
 780
 781     for (HttpRequestMethod m(METHOD_NONE); m != METHOD_ENUM_END; ++m) {
 782         if (m.isCacheble()) {
 783             if (StoreEntry *entry = storeGetPublic(url, m)) {
 784                 debugs(88, 5, "purging " << RequestMethodStr(m) << ' ' << url);
 785 #if USE_HTCP
 786                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 787                 if (m == METHOD_GET || m == METHOD_HEAD) {
 788                     get_or_head_sent = true;
 789                 }
 790 #endif
 791                 entry->release();
 792             }
 793         }
 794     }
 795
 796 #if USE_HTCP
 797     if (!get_or_head_sent) {
 798         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(METHOD_GET), HTCP_CLR_INVALIDATION);
 799     }
 800 #endif
 801 }
 802
 803 void
 804 clientReplyContext::purgeAllCached()
 805 {
 806     const char *url = urlCanonical(http->request);
 807     purgeEntriesByUrl(http->request, url);
 808 }
 809
 810 void
 811 clientReplyContext::created(StoreEntry *newEntry)
 812 {
 813     if (lookingforstore == 1)
 814         purgeFoundGet(newEntry);
 815     else if (lookingforstore == 2)
 816         purgeFoundHead(newEntry);
 817     else if (lookingforstore == 3)
 818         purgeDoPurgeGet(newEntry);
 819     else if (lookingforstore == 4)
 820         purgeDoPurgeHead(newEntry);
 821     else if (lookingforstore == 5)
 822         identifyFoundObject(newEntry);
 823 }
 824
 825 void
 826 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 827 {
 828     if (newEntry->isNull()) {
 829         lookingforstore = 2;
 830         StoreEntry::getPublicByRequestMethod(this, http->request, METHOD_HEAD);
 831     } else
 832         purgeFoundObject (newEntry);
 833 }
 834
 835 void
 836 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 837 {
 838     if (newEntry->isNull())
 839         purgeDoMissPurge();
 840     else
 841         purgeFoundObject (newEntry);
 842 }
 843
 844 void
 845 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 846 {
 847     assert (entry && !entry->isNull());
 848
 849     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 850         http->logType = LOG_TCP_DENIED;
 851         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL,
 852                                            http->getConn()->clientConnection->remote, http->request);
 853         startError(err);
 854         return;
 855     }
 856
 857     StoreIOBuffer localTempBuffer;
 858     /* Swap in the metadata */
 859     http->storeEntry(entry);
 860
 861     http->storeEntry()->lock();
 862     http->storeEntry()->createMemObject(http->uri, http->log_uri);
 863
 864     http->storeEntry()->mem_obj->method = http->request->method;
 865
 866     sc = storeClientListAdd(http->storeEntry(), this);
 867
 868     http->logType = LOG_TCP_HIT;
 869
 870     reqofs = 0;
 871
 872     localTempBuffer.offset = http->out.offset;
 873
 874     localTempBuffer.length = next()->readBuffer.length;
 875
 876     localTempBuffer.data = next()->readBuffer.data;
 877
 878     storeClientCopy(sc, http->storeEntry(),
 879                     localTempBuffer, CacheHit, this);
 880 }
 881
 882 void
 883 clientReplyContext::purgeRequest()
 884 {
 885     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 886            Config2.onoff.enable_purge);
 887
 888     if (!Config2.onoff.enable_purge) {
 889         http->logType = LOG_TCP_DENIED;
 890         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, http->getConn()->clientConnection->remote, http->request);
 891         startError(err);
 892         return;
 893     }
 894
 895     /* Release both IP cache */
 896     ipcacheInvalidate(http->request->GetHost());
 897
 898     if (!http->flags.purging)
 899         purgeRequestFindObjectToPurge();
 900     else
 901         purgeDoMissPurge();
 902 }
 903
 904 void
 905 clientReplyContext::purgeDoMissPurge()
 906 {
 907     http->logType = LOG_TCP_MISS;
 908     lookingforstore = 3;
 909     StoreEntry::getPublicByRequestMethod(this,http->request, METHOD_GET);
 910 }
 911
 912 void
 913 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
 914 {
 915     assert (newEntry);
 916     /* Move to new() when that is created */
 917     purgeStatus = HTTP_NOT_FOUND;
 918
 919     if (!newEntry->isNull()) {
 920         /* Release the cached URI */
 921         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
 922 #if USE_HTCP
 923         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(METHOD_GET), HTCP_CLR_PURGE);
 924 #endif
 925         newEntry->release();
 926         purgeStatus = HTTP_OK;
 927     }
 928
 929     lookingforstore = 4;
 930     StoreEntry::getPublicByRequestMethod(this, http->request, METHOD_HEAD);
 931 }
 932
 933 void
 934 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
 935 {
 936     if (newEntry && !newEntry->isNull()) {
 937         debugs(88, 4, "clientPurgeRequest: HEAD '" << newEntry->url() << "'" );
 938 #if USE_HTCP
 939         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(METHOD_HEAD), HTCP_CLR_PURGE);
 940 #endif
 941         newEntry->release();
 942         purgeStatus = HTTP_OK;
 943     }
 944
 945     /* And for Vary, release the base URI if none of the headers was included in the request */
 946
 947     if (http->request->vary_headers
 948             && !strstr(http->request->vary_headers, "=")) {
 949         StoreEntry *entry = storeGetPublic(urlCanonical(http->request), METHOD_GET);
 950
 951         if (entry) {
 952             debugs(88, 4, "clientPurgeRequest: Vary GET '" << entry->url() << "'" );
 953 #if USE_HTCP
 954             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(METHOD_GET), HTCP_CLR_PURGE);
 955 #endif
 956             entry->release();
 957             purgeStatus = HTTP_OK;
 958         }
 959
 960         entry = storeGetPublic(urlCanonical(http->request), METHOD_HEAD);
 961
 962         if (entry) {
 963             debugs(88, 4, "clientPurgeRequest: Vary HEAD '" << entry->url() << "'" );
 964 #if USE_HTCP
 965             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(METHOD_HEAD), HTCP_CLR_PURGE);
 966 #endif
 967             entry->release();
 968             purgeStatus = HTTP_OK;
 969         }
 970     }
 971
 972     /*
 973      * Make a new entry to hold the reply to be written
 974      * to the client.
 975      */
 976     /* FIXME: This doesn't need to go through the store. Simply
 977      * push down the client chain
 978      */
 979     createStoreEntry(http->request->method, request_flags());
 980
 981     triggerInitialStoreRead();
 982
 983     HttpReply *rep = new HttpReply;
 984     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
 985     http->storeEntry()->replaceHttpReply(rep);
 986     http->storeEntry()->complete();
 987 }
 988
 989 void
 990 clientReplyContext::traceReply(clientStreamNode * node)
 991 {
 992     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
 993     StoreIOBuffer localTempBuffer;
 994     createStoreEntry(http->request->method, request_flags());
 995     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
 996     localTempBuffer.length = nextNode->readBuffer.length;
 997     localTempBuffer.data = nextNode->readBuffer.data;
 998     storeClientCopy(sc, http->storeEntry(),
 999                     localTempBuffer, SendMoreData, this);
1000     http->storeEntry()->releaseRequest();
1001     http->storeEntry()->buffer();
1002     HttpReply *rep = new HttpReply;
1003     rep->setHeaders(HTTP_OK, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1004     http->storeEntry()->replaceHttpReply(rep);
1005     http->request->swapOut(http->storeEntry());
1006     http->storeEntry()->complete();
1007 }
1008
1009 #define SENDING_BODY 0
1010 #define SENDING_HDRSONLY 1
1011 int
1012 clientReplyContext::checkTransferDone()
1013 {
1014     StoreEntry *entry = http->storeEntry();
1015
1016     if (entry == NULL)
1017         return 0;
1018
1019     /*
1020      * For now, 'done_copying' is used for special cases like
1021      * Range and HEAD requests.
1022      */
1023     if (http->flags.done_copying)
1024         return 1;
1025
1026     if (http->request->flags.chunked_reply && !flags.complete) {
1027         // last-chunk was not sent
1028         return 0;
1029     }
1030
1031     /*
1032      * Handle STORE_OK objects.
1033      * objectLen(entry) will be set proprely.
1034      * RC: Does objectLen(entry) include the Headers?
1035      * RC: Yes.
1036      */
1037     if (entry->store_status == STORE_OK) {
1038         return storeOKTransferDone();
1039     } else {
1040         return storeNotOKTransferDone();
1041     }
1042 }
1043
1044 int
1045 clientReplyContext::storeOKTransferDone() const
1046 {
1047     assert(http->storeEntry()->objectLen() >= 0);
1048     assert(http->storeEntry()->objectLen() >= headers_sz);
1049     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1050         debugs(88,3,HERE << "storeOKTransferDone " <<
1051                " out.offset=" << http->out.offset <<
1052                " objectLen()=" << http->storeEntry()->objectLen() <<
1053                " headers_sz=" << headers_sz);
1054         return 1;
1055     }
1056
1057     return 0;
1058 }
1059
1060 int
1061 clientReplyContext::storeNotOKTransferDone() const
1062 {
1063     /*
1064      * Now, handle STORE_PENDING objects
1065      */
1066     MemObject *mem = http->storeEntry()->mem_obj;
1067     assert(mem != NULL);
1068     assert(http->request != NULL);
1069     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1070     HttpReply const *curReply = mem->getReply();
1071
1072     if (headers_sz == 0)
1073         /* haven't found end of headers yet */
1074         return 0;
1075
1076     /*
1077      * Figure out how much data we are supposed to send.
1078      * If we are sending a body and we don't have a content-length,
1079      * then we must wait for the object to become STORE_OK.
1080      */
1081     if (curReply->content_length < 0)
1082         return 0;
1083
1084     int64_t expectedLength = curReply->content_length + http->out.headers_sz;
1085
1086     if (http->out.size < expectedLength)
1087         return 0;
1088     else {
1089         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1090                " out.size=" << http->out.size <<
1091                " expectedLength=" << expectedLength);
1092         return 1;
1093     }
1094 }
1095
1096
1097 /* A write has completed, what is the next status based on the
1098  * canonical request data?
1099  * 1 something is wrong
1100  * 0 nothing is wrong.
1101  *
1102  */
1103 int
1104 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1105 {
1106 #if SIZEOF_INT64_T == 4
1107     if (http->out.size > 0x7FFF0000) {
1108         debugs(88, 1, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1109         debugs(88, 1, "\tclient " << http->getConn()->peer);
1110         debugs(88, 1, "\treceived " << http->out.size << " bytes");
1111         debugs(88, 1, "\tURI " << http->log_uri);
1112         return 1;
1113     }
1114
1115     if (http->out.offset > 0x7FFF0000) {
1116         debugs(88, 1, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1117         debugs(88, 1, "\tclient " << http->getConn()->peer);
1118         debugs(88, 1, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1119         debugs(88, 1, "\tURI " << http->log_uri);
1120         return 1;
1121     }
1122
1123 #endif
1124     return 0;
1125 }
1126
1127 /* Preconditions:
1128  * *http is a valid structure.
1129  * fd is either -1, or an open fd.
1130  *
1131  * TODO: enumify this
1132  *
1133  * This function is used by any http request sink, to determine the status
1134  * of the object.
1135  */
1136 clientStream_status_t
1137 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1138 {
1139     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1140     assert (context);
1141     assert (context->http == http);
1142     return context->replyStatus();
1143 }
1144
1145 clientStream_status_t
1146 clientReplyContext::replyStatus()
1147 {
1148     int done;
1149     /* Here because lower nodes don't need it */
1150
1151     if (http->storeEntry() == NULL) {
1152         debugs(88, 5, "clientReplyStatus: no storeEntry");
1153         return STREAM_FAILED;   /* yuck, but what can we do? */
1154     }
1155
1156     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1157         /* TODO: Could upstream read errors (result.flags.error) be
1158          * lost, and result in undersize requests being considered
1159          * complete. Should we tcp reset such connections ?
1160          */
1161         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1162         return STREAM_FAILED;
1163     }
1164
1165     if ((done = checkTransferDone()) != 0 || flags.complete) {
1166         debugs(88, 5, "clientReplyStatus: transfer is DONE");
1167         /* Ok we're finished, but how? */
1168
1169         const int64_t expectedBodySize =
1170             http->storeEntry()->getReply()->bodySize(http->request->method);
1171         if (!http->request->flags.proxy_keepalive && expectedBodySize < 0) {
1172             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
1173             return STREAM_FAILED;
1174         }
1175
1176         if (!done) {
1177             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1178             return STREAM_FAILED;
1179         }
1180
1181         if (expectedBodySize >= 0 && !http->gotEnough()) {
1182             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1183             return STREAM_UNPLANNED_COMPLETE;
1184         }
1185
1186         if (http->request->flags.proxy_keepalive) {
1187             debugs(88, 5, "clientReplyStatus: stream complete and can keepalive");
1188             return STREAM_COMPLETE;
1189         }
1190
1191         debugs(88, 5, "clientReplyStatus: stream was not expected to complete!");
1192         return STREAM_UNPLANNED_COMPLETE;
1193     }
1194
1195     // XXX: Should this be checked earlier? We could return above w/o checking.
1196     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1197         /* 4096 is a margin for the HTTP headers included in out.offset */
1198         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1199         return STREAM_FAILED;
1200     }
1201
1202     return STREAM_NONE;
1203 }
1204
1205 /* Responses with no body will not have a content-type header,
1206  * which breaks the rep_mime_type acl, which
1207  * coincidentally, is the most common acl for reply access lists.
1208  * A better long term fix for this is to allow acl matchs on the various
1209  * status codes, and then supply a default ruleset that puts these
1210  * codes before any user defines access entries. That way the user
1211  * can choose to block these responses where appropriate, but won't get
1212  * mysterious breakages.
1213  */
1214 bool
1215 clientReplyContext::alwaysAllowResponse(http_status sline) const
1216 {
1217     bool result;
1218
1219     switch (sline) {
1220
1221     case HTTP_CONTINUE:
1222
1223     case HTTP_SWITCHING_PROTOCOLS:
1224
1225     case HTTP_PROCESSING:
1226
1227     case HTTP_NO_CONTENT:
1228
1229     case HTTP_NOT_MODIFIED:
1230         result = true;
1231         break;
1232
1233     default:
1234         result = false;
1235     }
1236
1237     return result;
1238 }
1239
1240 /**
1241  * Generate the reply headers sent to client.
1242  *
1243  * Filters out unwanted entries and hop-by-hop from original reply header
1244  * then adds extra entries if we have more info than origin server
1245  * then adds Squid specific entries
1246  */
1247 void
1248 clientReplyContext::buildReplyHeader()
1249 {
1250     HttpHeader *hdr = &reply->header;
1251     int is_hit = logTypeIsATcpHit(http->logType);
1252     HttpRequest *request = http->request;
1253 #if DONT_FILTER_THESE
1254     /* but you might want to if you run Squid as an HTTP accelerator */
1255     /* hdr->delById(HDR_ACCEPT_RANGES); */
1256     hdr->delById(HDR_ETAG);
1257 #endif
1258
1259     if (is_hit)
1260         hdr->delById(HDR_SET_COOKIE);
1261     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1262
1263     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1264     // remove the Proxy-Authenticate header
1265     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0))
1266         reply->header.delById(HDR_PROXY_AUTHENTICATE);
1267
1268     reply->header.removeHopByHopEntries();
1269
1270     //    if (request->range)
1271     //      clientBuildRangeHeader(http, reply);
1272
1273     /*
1274      * Add a estimated Age header on cache hits.
1275      */
1276     if (is_hit) {
1277         /*
1278          * Remove any existing Age header sent by upstream caches
1279          * (note that the existing header is passed along unmodified
1280          * on cache misses)
1281          */
1282         hdr->delById(HDR_AGE);
1283         /*
1284          * This adds the calculated object age. Note that the details of the
1285          * age calculation is performed by adjusting the timestamp in
1286          * StoreEntry::timestampsSet(), not here.
1287          */
1288 #if DEAD_CODE
1289         // XXX: realy useless? or is there a bug now that this is detatched from the below if-sequence ?
1290         // looks like this pre-if was supposed to be the browser workaround...
1291         if (NULL == http->storeEntry())
1292             (void) 0;
1293         else if (http->storeEntry()->timestamp < 0)
1294             (void) 0;
1295 #endif
1296
1297         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1298             hdr->delById(HDR_DATE);
1299             hdr->insertTime(HDR_DATE, squid_curtime);
1300         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1301             // Swap the Date: header to current time if we are simulating an origin
1302             HttpHeaderEntry *h = hdr->findEntry(HDR_DATE);
1303             if (h)
1304                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1305             hdr->delById(HDR_DATE);
1306             hdr->insertTime(HDR_DATE, squid_curtime);
1307             h = hdr->findEntry(HDR_EXPIRES);
1308             if (h && http->storeEntry()->expires >= 0) {
1309                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1310                 hdr->delById(HDR_EXPIRES);
1311                 hdr->insertTime(HDR_EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1312             }
1313             if (http->storeEntry()->timestamp <= squid_curtime) {
1314                 // put X-Cache-Age: instead of Age:
1315                 char age[64];
1316                 snprintf(age, sizeof(age), "%ld", (long int) squid_curtime - http->storeEntry()->timestamp);
1317                 hdr->putExt("X-Cache-Age", age);
1318             }
1319         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1320             hdr->putInt(HDR_AGE,
1321                         squid_curtime - http->storeEntry()->timestamp);
1322             /* Signal old objects.  NB: rfc 2616 is not clear,
1323              * by implication, on whether we should do this to all
1324              * responses, or only cache hits.
1325              * 14.46 states it ONLY applys for heuristically caclulated
1326              * freshness values, 13.2.4 doesn't specify the same limitation.
1327              * We interpret RFC 2616 under the combination.
1328              */
1329             /* TODO: if maxage or s-maxage is present, don't do this */
1330
1331             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1332                 char tbuf[512];
1333                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1334                           "113", ThisCache,
1335                           "This cache hit is still fresh and more than 1 day old");
1336                 hdr->putStr(HDR_WARNING, tbuf);
1337             }
1338         }
1339     }
1340
1341     /* RFC 2616: Section 14.18
1342      *
1343      * Add a Date: header if missing.
1344      * We have access to a clock therefore are required to amend any shortcoming in servers.
1345      *
1346      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1347      */
1348     if ( !hdr->has(HDR_DATE) ) {
1349         if (!http->storeEntry())
1350             hdr->insertTime(HDR_DATE, squid_curtime);
1351         else if (http->storeEntry()->timestamp > 0)
1352             hdr->insertTime(HDR_DATE, http->storeEntry()->timestamp);
1353         else {
1354             debugs(88,DBG_IMPORTANT,"WARNING: An error inside Squid has caused an HTTP reply without Date:. Please report this:");
1355             /* dump something useful about the problem */
1356             http->storeEntry()->dump(DBG_IMPORTANT);
1357         }
1358     }
1359
1360     // add Warnings required by RFC 2616 if serving a stale hit
1361     if (http->request->flags.stale_if_hit && logTypeIsATcpHit(http->logType)) {
1362         hdr->putWarning(110, "Response is stale");
1363         if (http->request->flags.need_validation)
1364             hdr->putWarning(111, "Revalidation failed");
1365     }
1366
1367     /* Filter unproxyable authentication types */
1368     if (http->logType != LOG_TCP_DENIED &&
1369             hdr->has(HDR_WWW_AUTHENTICATE)) {
1370         HttpHeaderPos pos = HttpHeaderInitPos;
1371         HttpHeaderEntry *e;
1372
1373         int connection_auth_blocked = 0;
1374         while ((e = hdr->getEntry(&pos))) {
1375             if (e->id == HDR_WWW_AUTHENTICATE) {
1376                 const char *value = e->value.rawBuf();
1377
1378                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1379                         (value[4] == '\0' || value[4] == ' '))
1380                         ||
1381                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1382                          (value[9] == '\0' || value[9] == ' '))
1383                         ||
1384                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1385                          (value[8] == '\0' || value[8] == ' '))) {
1386                     if (request->flags.connection_auth_disabled) {
1387                         hdr->delAt(pos, connection_auth_blocked);
1388                         continue;
1389                     }
1390                     request->flags.must_keepalive = 1;
1391                     if (!request->flags.accelerated && !request->flags.intercepted) {
1392                         httpHeaderPutStrf(hdr, HDR_PROXY_SUPPORT, "Session-Based-Authentication");
1393                         /*
1394                           We send "[Proxy-]Connection: Proxy-Support" header to mark
1395                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1396                           understand the semantics of this header. The RFC should have included
1397                           this recommendation.
1398                         */
1399                         httpHeaderPutStrf(hdr, HDR_CONNECTION, "Proxy-support");
1400                     }
1401                     break;
1402                 }
1403             }
1404         }
1405
1406         if (connection_auth_blocked)
1407             hdr->refreshMask();
1408     }
1409
1410 #if USE_AUTH
1411     /* Handle authentication headers */
1412     if (http->logType == LOG_TCP_DENIED &&
1413             ( reply->sline.status == HTTP_PROXY_AUTHENTICATION_REQUIRED ||
1414               reply->sline.status == HTTP_UNAUTHORIZED)
1415        ) {
1416         /* Add authentication header */
1417         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1418          * depends on authenticate behaviour: all schemes to date send no extra
1419          * data on 407/401 responses, and do not check the accel state on 401/407
1420          * responses
1421          */
1422         authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1423     } else if (request->auth_user_request != NULL)
1424         authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1425 #endif
1426
1427     /* Append X-Cache */
1428     httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s",
1429                       is_hit ? "HIT" : "MISS", getMyHostname());
1430
1431 #if USE_CACHE_DIGESTS
1432     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1433     httpHeaderPutStrf(hdr, HDR_X_CACHE_LOOKUP, "%s from %s:%d",
1434                       lookup_type ? lookup_type : "NONE",
1435                       getMyHostname(), getMyPort());
1436
1437 #endif
1438
1439     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1440                                      (request->http_ver >= HttpVersion(1, 1));
1441
1442     /* Check whether we should send keep-alive */
1443     if (!Config.onoff.error_pconns && reply->sline.status >= 400 && !request->flags.must_keepalive) {
1444         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1445         request->flags.proxy_keepalive = 0;
1446     } else if (!Config.onoff.client_pconns && !request->flags.must_keepalive) {
1447         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1448         request->flags.proxy_keepalive = 0;
1449     } else if (request->flags.proxy_keepalive && shutting_down) {
1450         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1451         request->flags.proxy_keepalive = 0;
1452     } else if (request->flags.connection_auth && !reply->keep_alive) {
1453         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1454         request->flags.proxy_keepalive = 0;
1455     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1456         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1457         request->flags.proxy_keepalive = 0;
1458     } else if (fdUsageHigh()&& !request->flags.must_keepalive) {
1459         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1460         request->flags.proxy_keepalive = 0;
1461     }
1462
1463     // Decide if we send chunked reply
1464     if (maySendChunkedReply &&
1465             request->flags.proxy_keepalive &&
1466             reply->bodySize(request->method) < 0) {
1467         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1468         request->flags.chunked_reply = 1;
1469         hdr->putStr(HDR_TRANSFER_ENCODING, "chunked");
1470     }
1471
1472     /* Append VIA */
1473     if (Config.onoff.via) {
1474         LOCAL_ARRAY(char, bbuf, MAX_URL + 32);
1475         String strVia;
1476         hdr->getList(HDR_VIA, &strVia);
1477         snprintf(bbuf, MAX_URL + 32, "%d.%d %s",
1478                  reply->sline.version.major,
1479                  reply->sline.version.minor,
1480                  ThisCache);
1481         strListAdd(&strVia, bbuf, ',');
1482         hdr->delById(HDR_VIA);
1483         hdr->putStr(HDR_VIA, strVia.termedBuf());
1484     }
1485     /* Signal keep-alive or close explicitly */
1486     hdr->putStr(HDR_CONNECTION, request->flags.proxy_keepalive ? "keep-alive" : "close");
1487
1488 #if ADD_X_REQUEST_URI
1489     /*
1490      * Knowing the URI of the request is useful when debugging persistent
1491      * connections in a client; we cannot guarantee the order of http headers,
1492      * but X-Request-URI is likely to be the very last header to ease use from a
1493      * debugger [hdr->entries.count-1].
1494      */
1495     hdr->putStr(HDR_X_REQUEST_URI,
1496                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1497
1498 #endif
1499
1500     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1501     if ( hdr->has(HDR_SURROGATE_CONTROL) ) {
1502         if (!request->header.has(HDR_SURROGATE_CAPABILITY)) {
1503             hdr->delById(HDR_SURROGATE_CONTROL);
1504         }
1505         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1506     }
1507
1508     httpHdrMangleList(hdr, request, ROR_REPLY);
1509 }
1510
1511
1512 void
1513 clientReplyContext::cloneReply()
1514 {
1515     assert(reply == NULL);
1516
1517     HttpReply *rep = http->storeEntry()->getReply()->clone();
1518
1519     reply = HTTPMSGLOCK(rep);
1520
1521     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1522         /* RFC 2616 requires us to advertise our 1.1 version (but only on real HTTP traffic) */
1523         reply->sline.version = HttpVersion(1,1);
1524     }
1525
1526     /* do header conversions */
1527     buildReplyHeader();
1528 }
1529
1530 void
1531 clientReplyContext::identifyStoreObject()
1532 {
1533     HttpRequest *r = http->request;
1534
1535     if (r->flags.cachable || r->flags.internal) {
1536         lookingforstore = 5;
1537         StoreEntry::getPublicByRequest (this, r);
1538     } else {
1539         identifyFoundObject (NullStoreEntry::getInstance());
1540     }
1541 }
1542
1543 /**
1544  * Check state of the current StoreEntry object.
1545  * to see if we can determine the final status of the request.
1546  */
1547 void
1548 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1549 {
1550     StoreEntry *e = newEntry;
1551     HttpRequest *r = http->request;
1552
1553     /** \li If the entry received isNull() then we ignore it. */
1554     if (e->isNull()) {
1555         http->storeEntry(NULL);
1556     } else {
1557         http->storeEntry(e);
1558     }
1559
1560     e = http->storeEntry();
1561
1562     /* Release IP-cache entries on reload */
1563     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1564       * 'invalidate' the cached IP entries for this request ???
1565       */
1566     if (r->flags.nocache) {
1567
1568 #if USE_DNSSERVERS
1569
1570         ipcacheInvalidate(r->GetHost());
1571
1572 #else
1573
1574         ipcacheInvalidateNegative(r->GetHost());
1575
1576 #endif /* USE_DNSSERVERS */
1577
1578     }
1579
1580 #if USE_HTTP_VIOLATIONS
1581
1582     else if (r->flags.nocache_hack) {
1583
1584 #if USE_DNSSERVERS
1585
1586         ipcacheInvalidate(r->GetHost());
1587
1588 #else
1589
1590         ipcacheInvalidateNegative(r->GetHost());
1591
1592 #endif /* USE_DNSSERVERS */
1593
1594     }
1595
1596 #endif /* USE_HTTP_VIOLATIONS */
1597 #if USE_CACHE_DIGESTS
1598
1599     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1600
1601 #endif
1602
1603     if (NULL == http->storeEntry()) {
1604         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1605         debugs(85, 3, "clientProcessRequest2: StoreEntry is NULL -  MISS");
1606         http->logType = LOG_TCP_MISS;
1607         doGetMoreData();
1608         return;
1609     }
1610
1611     if (Config.onoff.offline) {
1612         /** \li If we are running in offline mode set to HIT */
1613         debugs(85, 3, "clientProcessRequest2: offline HIT");
1614         http->logType = LOG_TCP_HIT;
1615         doGetMoreData();
1616         return;
1617     }
1618
1619     if (http->redirect.status) {
1620         /** \li If redirection status is True force this to be a MISS */
1621         debugs(85, 3, "clientProcessRequest2: redirectStatus forced StoreEntry to NULL -  MISS");
1622         http->storeEntry(NULL);
1623         http->logType = LOG_TCP_MISS;
1624         doGetMoreData();
1625         return;
1626     }
1627
1628     if (!e->validToSend()) {
1629         debugs(85, 3, "clientProcessRequest2: !storeEntryValidToSend MISS" );
1630         http->storeEntry(NULL);
1631         http->logType = LOG_TCP_MISS;
1632         doGetMoreData();
1633         return;
1634     }
1635
1636     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1637         /* \li Special entries are always hits, no matter what the client says */
1638         debugs(85, 3, "clientProcessRequest2: ENTRY_SPECIAL HIT");
1639         http->logType = LOG_TCP_HIT;
1640         doGetMoreData();
1641         return;
1642     }
1643
1644     if (r->flags.nocache) {
1645         debugs(85, 3, "clientProcessRequest2: no-cache REFRESH MISS");
1646         http->storeEntry(NULL);
1647         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1648         doGetMoreData();
1649         return;
1650     }
1651
1652     debugs(85, 3, "clientProcessRequest2: default HIT");
1653     http->logType = LOG_TCP_HIT;
1654     doGetMoreData();
1655 }
1656
1657 /**
1658  * Request more data from the store for the client Stream
1659  * This is *the* entry point to this module.
1660  *
1661  * Preconditions:
1662  *  - This is the head of the list.
1663  *  - There is at least one more node.
1664  *  - Data context is not null
1665  */
1666 void
1667 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1668 {
1669     /* Test preconditions */
1670     assert(aNode != NULL);
1671     assert(cbdataReferenceValid(aNode));
1672     assert(aNode->node.prev == NULL);
1673     assert(aNode->node.next != NULL);
1674     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1675     assert (context);
1676     assert(context->http == http);
1677
1678
1679     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1680
1681     if (!context->ourNode)
1682         context->ourNode = aNode;
1683
1684     /* no cbdatareference, this is only used once, and safely */
1685     if (context->flags.storelogiccomplete) {
1686         StoreIOBuffer tempBuffer;
1687         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1688         tempBuffer.length = next->readBuffer.length;
1689         tempBuffer.data = next->readBuffer.data;
1690
1691         storeClientCopy(context->sc, http->storeEntry(),
1692                         tempBuffer, clientReplyContext::SendMoreData, context);
1693         return;
1694     }
1695
1696     if (context->http->request->method == METHOD_PURGE) {
1697         context->purgeRequest();
1698         return;
1699     }
1700
1701     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1702
1703     if (context->http->request->method == METHOD_TRACE) {
1704         if (context->http->request->header.getInt64(HDR_MAX_FORWARDS) == 0) {
1705             context->traceReply(aNode);
1706             return;
1707         }
1708
1709         /* continue forwarding, not finished yet. */
1710         http->logType = LOG_TCP_MISS;
1711
1712         context->doGetMoreData();
1713     } else
1714         context->identifyStoreObject();
1715 }
1716
1717 void
1718 clientReplyContext::doGetMoreData()
1719 {
1720     /* We still have to do store logic processing - vary, cache hit etc */
1721     if (http->storeEntry() != NULL) {
1722         /* someone found the object in the cache for us */
1723         StoreIOBuffer localTempBuffer;
1724
1725         http->storeEntry()->lock();
1726
1727         if (http->storeEntry()->mem_obj == NULL) {
1728             /*
1729              * This if-block exists because we don't want to clobber
1730              * a preexiting mem_obj->method value if the mem_obj
1731              * already exists.  For example, when a HEAD request
1732              * is a cache hit for a GET response, we want to keep
1733              * the method as GET.
1734              */
1735             http->storeEntry()->createMemObject(http->uri, http->log_uri);
1736             http->storeEntry()->mem_obj->method = http->request->method;
1737         }
1738
1739         sc = storeClientListAdd(http->storeEntry(), this);
1740 #if USE_DELAY_POOLS
1741         sc->setDelayId(DelayId::DelayClient(http));
1742 #endif
1743
1744         assert(http->logType == LOG_TCP_HIT);
1745         reqofs = 0;
1746         /* guarantee nothing has been sent yet! */
1747         assert(http->out.size == 0);
1748         assert(http->out.offset == 0);
1749
1750         if (Ip::Qos::TheConfig.isHitTosActive()) {
1751             Ip::Qos::doTosLocalHit(http->getConn()->clientConnection);
1752         }
1753
1754         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1755             Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection);
1756         }
1757
1758         localTempBuffer.offset = reqofs;
1759         localTempBuffer.length = getNextNode()->readBuffer.length;
1760         localTempBuffer.data = getNextNode()->readBuffer.data;
1761         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1762     } else {
1763         /* MISS CASE, http->logType is already set! */
1764         processMiss();
1765     }
1766 }
1767
1768 /** The next node has removed itself from the stream. */
1769 void
1770 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1771 {
1772     /** detach from the stream */
1773     clientStreamDetach(node, http);
1774 }
1775
1776 /**
1777  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1778  * such, writes processed message to the message recipient
1779  */
1780 void
1781 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1782 {
1783     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1784     context->sendMoreData (result);
1785 }
1786
1787 void
1788 clientReplyContext::makeThisHead()
1789 {
1790     /* At least, I think thats what this does */
1791     dlinkDelete(&http->active, &ClientActiveRequests);
1792     dlinkAdd(http, &http->active, &ClientActiveRequests);
1793 }
1794
1795 bool
1796 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1797 {
1798     return /* aborted request */
1799         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1800         /* Upstream read error */ (result.flags.error) ||
1801         /* Upstream EOF */ (sizeToProcess == 0);
1802 }
1803
1804 void
1805 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1806 {
1807     /** call clientWriteComplete so the client socket gets closed
1808      *
1809      * We call into the stream, because we don't know that there is a
1810      * client socket!
1811      */
1812     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1813     StoreIOBuffer localTempBuffer;
1814     flags.complete = 1;
1815     http->request->flags.stream_error = 1;
1816     localTempBuffer.flags.error = result.flags.error;
1817     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1818                          localTempBuffer);
1819 }
1820
1821 void
1822 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1823 {
1824     StoreIOBuffer localTempBuffer;
1825
1826     if (result.length == 0) {
1827         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1828         flags.complete = 1;
1829     }
1830
1831     assert(result.offset - headers_sz == next()->readBuffer.offset);
1832     localTempBuffer.offset = result.offset - headers_sz;
1833     localTempBuffer.length = result.length;
1834
1835     if (localTempBuffer.length)
1836         localTempBuffer.data = source;
1837
1838     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1839                          localTempBuffer);
1840 }
1841
1842 clientStreamNode *
1843 clientReplyContext::next() const
1844 {
1845     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1846     return getNextNode();
1847 }
1848
1849 void
1850 clientReplyContext::sendBodyTooLargeError()
1851 {
1852     Ip::Address tmp_noaddr;
1853     tmp_noaddr.SetNoAddr(); // TODO: make a global const
1854     http->logType = LOG_TCP_DENIED_REPLY;
1855     ErrorState *err = clientBuildError(ERR_TOO_BIG, HTTP_FORBIDDEN, NULL,
1856                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1857                                        http->request);
1858     removeClientStoreReference(&(sc), http);
1859     HTTPMSGUNLOCK(reply);
1860     startError(err);
1861
1862 }
1863
1864 /// send 412 (Precondition Failed) to client
1865 void
1866 clientReplyContext::sendPreconditionFailedError()
1867 {
1868     http->logType = LOG_TCP_HIT;
1869     ErrorState *const err =
1870         clientBuildError(ERR_PRECONDITION_FAILED, HTTP_PRECONDITION_FAILED,
1871                          NULL, http->getConn()->clientConnection->remote, http->request);
1872     removeClientStoreReference(&sc, http);
1873     HTTPMSGUNLOCK(reply);
1874     startError(err);
1875 }
1876
1877 /// send 304 (Not Modified) to client
1878 void
1879 clientReplyContext::sendNotModified()
1880 {
1881     StoreEntry *e = http->storeEntry();
1882     const time_t timestamp = e->timestamp;
1883     HttpReply *const temprep = e->getReply()->make304();
1884     http->logType = LOG_TCP_IMS_HIT;
1885     removeClientStoreReference(&sc, http);
1886     createStoreEntry(http->request->method, request_flags());
1887     e = http->storeEntry();
1888     // Copy timestamp from the original entry so the 304
1889     // reply has a meaningful Age: header.
1890     e->timestampsSet();
1891     e->timestamp = timestamp;
1892     e->replaceHttpReply(temprep);
1893     e->complete();
1894     /*
1895      * TODO: why put this in the store and then serialise it and
1896      * then parse it again. Simply mark the request complete in
1897      * our context and write the reply struct to the client side.
1898      */
1899     triggerInitialStoreRead();
1900 }
1901
1902 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1903 /// depending on request method
1904 void
1905 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1906 {
1907     if (http->request->method == METHOD_GET ||
1908             http->request->method == METHOD_HEAD)
1909         sendNotModified();
1910     else
1911         sendPreconditionFailedError();
1912 }
1913
1914 void
1915 clientReplyContext::processReplyAccess ()
1916 {
1917     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
1918     assert(reply);
1919
1920     /** Don't block our own responses or HTTP status messages */
1921     if (http->logType == LOG_TCP_DENIED ||
1922             http->logType == LOG_TCP_DENIED_REPLY ||
1923             alwaysAllowResponse(reply->sline.status)) {
1924         headers_sz = reply->hdr_sz;
1925         processReplyAccessResult(ACCESS_ALLOWED);
1926         return;
1927     }
1928
1929     /** Check for reply to big error */
1930     if (reply->expectedBodyTooLarge(*http->request)) {
1931         sendBodyTooLargeError();
1932         return;
1933     }
1934
1935     headers_sz = reply->hdr_sz;
1936
1937     /** check for absent access controls (permit by default) */
1938     if (!Config.accessList.reply) {
1939         processReplyAccessResult(ACCESS_ALLOWED);
1940         return;
1941     }
1942
1943     /** Process http_reply_access lists */
1944     ACLFilledChecklist *replyChecklist =
1945         clientAclChecklistCreate(Config.accessList.reply, http);
1946     replyChecklist->reply = HTTPMSGLOCK(reply);
1947     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
1948 }
1949
1950 void
1951 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
1952 {
1953     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
1954     me->processReplyAccessResult(rv);
1955 }
1956
1957 void
1958 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
1959 {
1960     debugs(88, 2, "The reply for " << RequestMethodStr(http->request->method)
1961            << " " << http->uri << " is " << accessAllowed << ", because it matched '"
1962            << (AclMatchedName ? AclMatchedName : "NO ACL's") << "'" );
1963
1964     if (accessAllowed != ACCESS_ALLOWED) {
1965         ErrorState *err;
1966         err_type page_id;
1967         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
1968
1969         http->logType = LOG_TCP_DENIED_REPLY;
1970
1971         if (page_id == ERR_NONE)
1972             page_id = ERR_ACCESS_DENIED;
1973
1974         Ip::Address tmp_noaddr;
1975         tmp_noaddr.SetNoAddr();
1976         err = clientBuildError(page_id, HTTP_FORBIDDEN, NULL,
1977                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1978                                http->request);
1979
1980         removeClientStoreReference(&sc, http);
1981
1982         HTTPMSGUNLOCK(reply);
1983
1984         startError(err);
1985
1986
1987         return;
1988     }
1989
1990     /* Ok, the reply is allowed, */
1991     http->loggingEntry(http->storeEntry());
1992
1993     ssize_t body_size = reqofs - reply->hdr_sz;
1994     if (body_size < 0) {
1995         reqofs = reply->hdr_sz;
1996         body_size = 0;
1997     }
1998
1999     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2000            (int) body_size << " bytes after " << reply->hdr_sz <<
2001            " bytes of headers");
2002
2003 #if USE_SQUID_ESI
2004
2005     if (http->flags.accel && reply->sline.status != HTTP_FORBIDDEN &&
2006             !alwaysAllowResponse(reply->sline.status) &&
2007             esiEnableProcessing(reply)) {
2008         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2009         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2010                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2011     }
2012
2013 #endif
2014
2015     if (http->request->method == METHOD_HEAD) {
2016         /* do not forward body for HEAD replies */
2017         body_size = 0;
2018         http->flags.done_copying = 1;
2019         flags.complete = 1;
2020     }
2021
2022     assert (!flags.headersSent);
2023     flags.headersSent = true;
2024
2025     StoreIOBuffer localTempBuffer;
2026     char *buf = next()->readBuffer.data;
2027     char *body_buf = buf + reply->hdr_sz;
2028
2029     //Server side may disable ranges under some circumstances.
2030
2031     if ((!http->request->range))
2032         next()->readBuffer.offset = 0;
2033
2034     body_buf -= next()->readBuffer.offset;
2035
2036     if (next()->readBuffer.offset != 0) {
2037         if (next()->readBuffer.offset > body_size) {
2038             /* Can't use any of the body we received. send nothing */
2039             localTempBuffer.length = 0;
2040             localTempBuffer.data = NULL;
2041         } else {
2042             localTempBuffer.length = body_size - next()->readBuffer.offset;
2043             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2044         }
2045     } else {
2046         localTempBuffer.length = body_size;
2047         localTempBuffer.data = body_buf;
2048     }
2049
2050     /* TODO??: move the data in the buffer back by the request header size */
2051     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2052                          http, reply, localTempBuffer);
2053
2054     return;
2055 }
2056
2057 void
2058 clientReplyContext::sendMoreData (StoreIOBuffer result)
2059 {
2060     if (deleting)
2061         return;
2062
2063     StoreEntry *entry = http->storeEntry();
2064
2065     ConnStateData * conn = http->getConn();
2066
2067     if (conn == NULL || !conn->isOpen()) {
2068         // too late, our conn is closing
2069         // TODO: should we also quit?
2070         debugs(33,3, HERE << "not sending more data to a closing " << conn->clientConnection);
2071         return;
2072     }
2073
2074     char *buf = next()->readBuffer.data;
2075
2076     if (buf != result.data) {
2077         /* we've got to copy some data */
2078         assert(result.length <= next()->readBuffer.length);
2079         memcpy(buf, result.data, result.length);
2080     }
2081
2082     if (reqofs==0 && !logTypeIsATcpHit(http->logType) && Comm::IsConnOpen(conn->clientConnection)) {
2083         if (Ip::Qos::TheConfig.isHitTosActive()) {
2084             Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2085         }
2086         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2087             Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2088         }
2089     }
2090
2091     /* We've got the final data to start pushing... */
2092     flags.storelogiccomplete = 1;
2093
2094     reqofs += result.length;
2095
2096     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2097
2098     assert(http->request != NULL);
2099
2100     /* ESI TODO: remove this assert once everything is stable */
2101     assert(http->client_stream.head->data
2102            && cbdataReferenceValid(http->client_stream.head->data));
2103
2104     makeThisHead();
2105
2106     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2107            reqofs << " bytes (" << result.length <<
2108            " new bytes)");
2109     debugs(88, 5, "clientReplyContext::sendMoreData:"
2110            << conn->clientConnection <<
2111            " '" << entry->url() << "'" <<
2112            " out.offset=" << http->out.offset);
2113
2114     /* update size of the request */
2115     reqsize = reqofs;
2116
2117     if (errorInStream(result, reqofs)) {
2118         sendStreamError(result);
2119         return;
2120     }
2121
2122     if (flags.headersSent) {
2123         pushStreamData (result, buf);
2124         return;
2125     }
2126
2127     cloneReply();
2128
2129     /* handle headers */
2130
2131     if (Config.onoff.log_mime_hdrs) {
2132         size_t k;
2133
2134         if ((k = headersEnd(buf, reqofs))) {
2135             safe_free(http->al.headers.reply);
2136             http->al.headers.reply = (char *)xcalloc(k + 1, 1);
2137             xstrncpy(http->al.headers.reply, buf, k);
2138         }
2139     }
2140
2141     holdingBuffer = result;
2142     processReplyAccess();
2143     return;
2144 }
2145
2146
2147
2148 /* Using this breaks the client layering just a little!
2149  */
2150 void
2151 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, request_flags reqFlags)
2152 {
2153     assert(http != NULL);
2154     /*
2155      * For erroneous requests, we might not have a h->request,
2156      * so make a fake one.
2157      */
2158
2159     if (http->request == NULL)
2160         http->request = HTTPMSGLOCK(new HttpRequest(m, AnyP::PROTO_NONE, null_string));
2161
2162     StoreEntry *e = storeCreateEntry(http->uri, http->log_uri, reqFlags, m);
2163
2164     sc = storeClientListAdd(e, this);
2165
2166 #if USE_DELAY_POOLS
2167     sc->setDelayId(DelayId::DelayClient(http));
2168 #endif
2169
2170     reqofs = 0;
2171
2172     reqsize = 0;
2173
2174     /* I don't think this is actually needed! -- adrian */
2175     /* http->reqbuf = http->norm_reqbuf; */
2176     //    assert(http->reqbuf == http->norm_reqbuf);
2177     /* The next line is illegal because we don't know if the client stream
2178      * buffers have been set up
2179      */
2180     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2181     //        SendMoreData, this);
2182     /* So, we mark the store logic as complete */
2183     flags.storelogiccomplete = 1;
2184
2185     /* and get the caller to request a read, from whereever they are */
2186     /* NOTE: after ANY data flows down the pipe, even one step,
2187      * this function CAN NOT be used to manage errors
2188      */
2189     http->storeEntry(e);
2190 }
2191
2192 ErrorState *
2193 clientBuildError(err_type page_id, http_status status, char const *url,
2194                  Ip::Address &src_addr, HttpRequest * request)
2195 {
2196     ErrorState *err = errorCon(page_id, status, request);
2197     err->src_addr = src_addr;
2198
2199     if (url)
2200         err->url = xstrdup(url);
2201
2202     return err;
2203 }