src/http.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 /*
  37  * Anonymizing patch by lutz@as-node.jena.thur.de
  38  * have a look into http-anon.c to get more informations.
  39  */
  40
  41 #include "squid-old.h"
  42
  43 #include "acl/FilledChecklist.h"
  44 #if USE_AUTH
  45 #include "auth/UserRequest.h"
  46 #endif
  47 #include "base/AsyncJobCalls.h"
  48 #include "base/TextException.h"
  49 #include "base64.h"
  50 #include "comm/Connection.h"
  51 #include "comm/Write.h"
  52 #if USE_DELAY_POOLS
  53 #include "DelayPools.h"
  54 #endif
  55 #include "err_detail_type.h"
  56 #include "errorpage.h"
  57 #include "fde.h"
  58 #include "http.h"
  59 #include "HttpControlMsg.h"
  60 #include "HttpHdrContRange.h"
  61 #include "HttpHdrCc.h"
  62 #include "HttpHdrSc.h"
  63 #include "HttpHdrScTarget.h"
  64 #include "HttpReply.h"
  65 #include "HttpRequest.h"
  66 #include "MemBuf.h"
  67 #include "MemObject.h"
  68 #include "protos.h"
  69 #include "rfc1738.h"
  70 #include "SquidTime.h"
  71 #include "StatCounters.h"
  72 #include "Store.h"
  73
  74
  75 #define SQUID_ENTER_THROWING_CODE() try {
  76 #define SQUID_EXIT_THROWING_CODE(status) \
  77         status = true; \
  78     } \
  79     catch (const std::exception &e) { \
  80         debugs (11, 1, "Exception error:" << e.what()); \
  81         status = false; \
  82     }
  83
  84 CBDATA_CLASS_INIT(HttpStateData);
  85
  86 static const char *const crlf = "\r\n";
  87
  88 static void httpMaybeRemovePublic(StoreEntry *, http_status);
  89 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request,
  90         HttpHeader * hdr_out, const int we_do_ranges, const http_state_flags);
  91
  92 HttpStateData::HttpStateData(FwdState *theFwdState) : AsyncJob("HttpStateData"), ServerStateData(theFwdState),
  93         lastChunk(0), header_bytes_read(0), reply_bytes_read(0),
  94         body_bytes_truncated(0), httpChunkDecoder(NULL)
  95 {
  96     debugs(11,5,HERE << "HttpStateData " << this << " created");
  97     ignoreCacheControl = false;
  98     surrogateNoStore = false;
  99     serverConnection = fwd->serverConnection();
 100     readBuf = new MemBuf;
 101     readBuf->init(16*1024, 256*1024);
 102
 103     // reset peer response time stats for %<pt
 104     request->hier.peer_http_request_sent.tv_sec = 0;
 105     request->hier.peer_http_request_sent.tv_usec = 0;
 106
 107     if (fwd->serverConnection() != NULL)
 108         _peer = cbdataReference(fwd->serverConnection()->getPeer());         /* might be NULL */
 109
 110     if (_peer) {
 111         request->flags.proxying = 1;
 112         /*
 113          * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
 114          * We might end up getting the object from somewhere else if,
 115          * for example, the request to this neighbor fails.
 116          */
 117         if (_peer->options.proxy_only)
 118             entry->releaseRequest();
 119
 120 #if USE_DELAY_POOLS
 121         entry->setNoDelay(_peer->options.no_delay);
 122 #endif
 123     }
 124
 125     /*
 126      * register the handler to free HTTP state data when the FD closes
 127      */
 128     typedef CommCbMemFunT<HttpStateData, CommCloseCbParams> Dialer;
 129     closeHandler = JobCallback(9, 5, Dialer, this, HttpStateData::httpStateConnClosed);
 130     comm_add_close_handler(serverConnection->fd, closeHandler);
 131 }
 132
 133 HttpStateData::~HttpStateData()
 134 {
 135     /*
 136      * don't forget that ~ServerStateData() gets called automatically
 137      */
 138
 139     if (!readBuf->isNull())
 140         readBuf->clean();
 141
 142     delete readBuf;
 143
 144     if (httpChunkDecoder)
 145         delete httpChunkDecoder;
 146
 147     cbdataReferenceDone(_peer);
 148
 149     debugs(11,5, HERE << "HttpStateData " << this << " destroyed; " << serverConnection);
 150 }
 151
 152 const Comm::ConnectionPointer &
 153 HttpStateData::dataConnection() const
 154 {
 155     return serverConnection;
 156 }
 157
 158 void
 159 HttpStateData::httpStateConnClosed(const CommCloseCbParams &params)
 160 {
 161     debugs(11, 5, "httpStateFree: FD " << params.fd << ", httpState=" << params.data);
 162     mustStop("HttpStateData::httpStateConnClosed");
 163 }
 164
 165 int
 166 httpCachable(const HttpRequestMethod& method)
 167 {
 168     /* GET and HEAD are cachable. Others are not. */
 169
 170     // TODO: replase to HttpRequestMethod::isCachable() ?
 171     if (method != METHOD_GET && method != METHOD_HEAD)
 172         return 0;
 173
 174     /* else cachable */
 175     return 1;
 176 }
 177
 178 void
 179 HttpStateData::httpTimeout(const CommTimeoutCbParams &params)
 180 {
 181     debugs(11, 4, HERE << serverConnection << ": '" << entry->url() << "'" );
 182
 183     if (entry->store_status == STORE_PENDING) {
 184         fwd->fail(new ErrorState(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT, fwd->request));
 185     }
 186
 187     serverConnection->close();
 188 }
 189
 190 static void
 191 httpMaybeRemovePublic(StoreEntry * e, http_status status)
 192 {
 193     int remove = 0;
 194     int forbidden = 0;
 195     StoreEntry *pe;
 196
 197     if (!EBIT_TEST(e->flags, KEY_PRIVATE))
 198         return;
 199
 200     switch (status) {
 201
 202     case HTTP_OK:
 203
 204     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 205
 206     case HTTP_MULTIPLE_CHOICES:
 207
 208     case HTTP_MOVED_PERMANENTLY:
 209
 210     case HTTP_MOVED_TEMPORARILY:
 211
 212     case HTTP_GONE:
 213
 214     case HTTP_NOT_FOUND:
 215         remove = 1;
 216
 217         break;
 218
 219     case HTTP_FORBIDDEN:
 220
 221     case HTTP_METHOD_NOT_ALLOWED:
 222         forbidden = 1;
 223
 224         break;
 225
 226 #if WORK_IN_PROGRESS
 227
 228     case HTTP_UNAUTHORIZED:
 229         forbidden = 1;
 230
 231         break;
 232
 233 #endif
 234
 235     default:
 236 #if QUESTIONABLE
 237         /*
 238          * Any 2xx response should eject previously cached entities...
 239          */
 240
 241         if (status >= 200 && status < 300)
 242             remove = 1;
 243
 244 #endif
 245
 246         break;
 247     }
 248
 249     if (!remove && !forbidden)
 250         return;
 251
 252     assert(e->mem_obj);
 253
 254     if (e->mem_obj->request)
 255         pe = storeGetPublicByRequest(e->mem_obj->request);
 256     else
 257         pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method);
 258
 259     if (pe != NULL) {
 260         assert(e != pe);
 261 #if USE_HTCP
 262         neighborsHtcpClear(e, NULL, e->mem_obj->request, e->mem_obj->method, HTCP_CLR_INVALIDATION);
 263 #endif
 264         pe->release();
 265     }
 266
 267     /** \par
 268      * Also remove any cached HEAD response in case the object has
 269      * changed.
 270      */
 271     if (e->mem_obj->request)
 272         pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD);
 273     else
 274         pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD);
 275
 276     if (pe != NULL) {
 277         assert(e != pe);
 278 #if USE_HTCP
 279         neighborsHtcpClear(e, NULL, e->mem_obj->request, HttpRequestMethod(METHOD_HEAD), HTCP_CLR_INVALIDATION);
 280 #endif
 281         pe->release();
 282     }
 283 }
 284
 285 void
 286 HttpStateData::processSurrogateControl(HttpReply *reply)
 287 {
 288     if (request->flags.accelerated && reply->surrogate_control) {
 289         HttpHdrScTarget *sctusable = reply->surrogate_control->getMergedTarget(Config.Accel.surrogate_id);
 290
 291         if (sctusable) {
 292             if (sctusable->noStore() ||
 293                     (Config.onoff.surrogate_is_remote
 294                      && sctusable->noStoreRemote())) {
 295                 surrogateNoStore = true;
 296                 entry->makePrivate();
 297             }
 298
 299             /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
 300              * accelerated request or not...
 301              * Still, this is an abstraction breach. - RC
 302              */
 303             if (sctusable->hasMaxAge()) {
 304                 if (sctusable->maxAge() < sctusable->maxStale())
 305                     reply->expires = reply->date + sctusable->maxAge();
 306                 else
 307                     reply->expires = reply->date + sctusable->maxStale();
 308
 309                 /* And update the timestamps */
 310                 entry->timestampsSet();
 311             }
 312
 313             /* We ignore cache-control directives as per the Surrogate specification */
 314             ignoreCacheControl = true;
 315
 316             delete sctusable;
 317         }
 318     }
 319 }
 320
 321 int
 322 HttpStateData::cacheableReply()
 323 {
 324     HttpReply const *rep = finalReply();
 325     HttpHeader const *hdr = &rep->header;
 326     const char *v;
 327 #if USE_HTTP_VIOLATIONS
 328
 329     const refresh_t *R = NULL;
 330
 331     /* This strange looking define first looks up the refresh pattern
 332      * and then checks if the specified flag is set. The main purpose
 333      * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
 334      * condition
 335      */
 336 #define REFRESH_OVERRIDE(flag) \
 337     ((R = (R ? R : refreshLimits(entry->mem_obj->url))) , \
 338     (R && R->flags.flag))
 339 #else
 340 #define REFRESH_OVERRIDE(flag) 0
 341 #endif
 342
 343     if (surrogateNoStore)
 344         return 0;
 345
 346     // RFC 2616: do not cache replies to responses with no-store CC directive
 347     if (request && request->cache_control &&
 348             request->cache_control->noStore() &&
 349             !REFRESH_OVERRIDE(ignore_no_store))
 350         return 0;
 351
 352     if (!ignoreCacheControl && request->cache_control != NULL) {
 353         const HttpHdrCc* cc=request->cache_control;
 354         if (cc->Private()) {
 355             if (!REFRESH_OVERRIDE(ignore_private))
 356                 return 0;
 357         }
 358
 359         if (cc->noCache()) {
 360             if (!REFRESH_OVERRIDE(ignore_no_cache))
 361                 return 0;
 362         }
 363
 364         if (cc->noStore()) {
 365             if (!REFRESH_OVERRIDE(ignore_no_store))
 366                 return 0;
 367         }
 368     }
 369
 370     if (request->flags.auth || request->flags.auth_sent) {
 371         /*
 372          * Responses to requests with authorization may be cached
 373          * only if a Cache-Control: public reply header is present.
 374          * RFC 2068, sec 14.9.4
 375          */
 376
 377         if (!request->cache_control || !request->cache_control->Public()) {
 378             if (!REFRESH_OVERRIDE(ignore_auth))
 379                 return 0;
 380         }
 381     }
 382
 383     /* Pragma: no-cache in _replies_ is not documented in HTTP,
 384      * but servers like "Active Imaging Webcast/2.0" sure do use it */
 385     if (hdr->has(HDR_PRAGMA)) {
 386         String s = hdr->getList(HDR_PRAGMA);
 387         const int no_cache = strListIsMember(&s, "no-cache", ',');
 388         s.clean();
 389
 390         if (no_cache) {
 391             if (!REFRESH_OVERRIDE(ignore_no_cache))
 392                 return 0;
 393         }
 394     }
 395
 396     /*
 397      * The "multipart/x-mixed-replace" content type is used for
 398      * continuous push replies.  These are generally dynamic and
 399      * probably should not be cachable
 400      */
 401     if ((v = hdr->getStr(HDR_CONTENT_TYPE)))
 402         if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
 403             return 0;
 404
 405     switch (rep->sline.status) {
 406         /* Responses that are cacheable */
 407
 408     case HTTP_OK:
 409
 410     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 411
 412     case HTTP_MULTIPLE_CHOICES:
 413
 414     case HTTP_MOVED_PERMANENTLY:
 415
 416     case HTTP_GONE:
 417         /*
 418          * Don't cache objects that need to be refreshed on next request,
 419          * unless we know how to refresh it.
 420          */
 421
 422         if (!refreshIsCachable(entry) && !REFRESH_OVERRIDE(store_stale)) {
 423             debugs(22, 3, "refreshIsCachable() returned non-cacheable..");
 424             return 0;
 425         } else
 426             return 1;
 427
 428         /* NOTREACHED */
 429         break;
 430
 431         /* Responses that only are cacheable if the server says so */
 432
 433     case HTTP_MOVED_TEMPORARILY:
 434     case HTTP_TEMPORARY_REDIRECT:
 435         if (rep->expires > rep->date && rep->date > 0)
 436             return 1;
 437         else
 438             return 0;
 439
 440         /* NOTREACHED */
 441         break;
 442
 443         /* Errors can be negatively cached */
 444
 445     case HTTP_NO_CONTENT:
 446
 447     case HTTP_USE_PROXY:
 448
 449     case HTTP_BAD_REQUEST:
 450
 451     case HTTP_FORBIDDEN:
 452
 453     case HTTP_NOT_FOUND:
 454
 455     case HTTP_METHOD_NOT_ALLOWED:
 456
 457     case HTTP_REQUEST_URI_TOO_LARGE:
 458
 459     case HTTP_INTERNAL_SERVER_ERROR:
 460
 461     case HTTP_NOT_IMPLEMENTED:
 462
 463     case HTTP_BAD_GATEWAY:
 464
 465     case HTTP_SERVICE_UNAVAILABLE:
 466
 467     case HTTP_GATEWAY_TIMEOUT:
 468         return -1;
 469
 470         /* NOTREACHED */
 471         break;
 472
 473         /* Some responses can never be cached */
 474
 475     case HTTP_PARTIAL_CONTENT:  /* Not yet supported */
 476
 477     case HTTP_SEE_OTHER:
 478
 479     case HTTP_NOT_MODIFIED:
 480
 481     case HTTP_UNAUTHORIZED:
 482
 483     case HTTP_PROXY_AUTHENTICATION_REQUIRED:
 484
 485     case HTTP_INVALID_HEADER:   /* Squid header parsing error */
 486
 487     case HTTP_HEADER_TOO_LARGE:
 488
 489     case HTTP_PAYMENT_REQUIRED:
 490     case HTTP_NOT_ACCEPTABLE:
 491     case HTTP_REQUEST_TIMEOUT:
 492     case HTTP_CONFLICT:
 493     case HTTP_LENGTH_REQUIRED:
 494     case HTTP_PRECONDITION_FAILED:
 495     case HTTP_REQUEST_ENTITY_TOO_LARGE:
 496     case HTTP_UNSUPPORTED_MEDIA_TYPE:
 497     case HTTP_UNPROCESSABLE_ENTITY:
 498     case HTTP_LOCKED:
 499     case HTTP_FAILED_DEPENDENCY:
 500     case HTTP_INSUFFICIENT_STORAGE:
 501     case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
 502     case HTTP_EXPECTATION_FAILED:
 503
 504         return 0;
 505
 506     default:
 507         /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
 508         debugs (11, 3, HERE << "Unknown HTTP status code " << rep->sline.status << ". Not cacheable.");
 509
 510         return 0;
 511
 512         /* NOTREACHED */
 513         break;
 514     }
 515
 516     /* NOTREACHED */
 517 }
 518
 519 /*
 520  * For Vary, store the relevant request headers as
 521  * virtual headers in the reply
 522  * Returns false if the variance cannot be stored
 523  */
 524 const char *
 525 httpMakeVaryMark(HttpRequest * request, HttpReply const * reply)
 526 {
 527     String vary, hdr;
 528     const char *pos = NULL;
 529     const char *item;
 530     const char *value;
 531     int ilen;
 532     static String vstr;
 533
 534     vstr.clean();
 535     vary = reply->header.getList(HDR_VARY);
 536
 537     while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
 538         char *name = (char *)xmalloc(ilen + 1);
 539         xstrncpy(name, item, ilen + 1);
 540         Tolower(name);
 541
 542         if (strcmp(name, "*") == 0) {
 543             /* Can not handle "Vary: *" withtout ETag support */
 544             safe_free(name);
 545             vstr.clean();
 546             break;
 547         }
 548
 549         strListAdd(&vstr, name, ',');
 550         hdr = request->header.getByName(name);
 551         safe_free(name);
 552         value = hdr.termedBuf();
 553
 554         if (value) {
 555             value = rfc1738_escape_part(value);
 556             vstr.append("=\"", 2);
 557             vstr.append(value);
 558             vstr.append("\"", 1);
 559         }
 560
 561         hdr.clean();
 562     }
 563
 564     vary.clean();
 565 #if X_ACCELERATOR_VARY
 566
 567     pos = NULL;
 568     vary = reply->header.getList(HDR_X_ACCELERATOR_VARY);
 569
 570     while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
 571         char *name = (char *)xmalloc(ilen + 1);
 572         xstrncpy(name, item, ilen + 1);
 573         Tolower(name);
 574         strListAdd(&vstr, name, ',');
 575         hdr = request->header.getByName(name);
 576         safe_free(name);
 577         value = hdr.termedBuf();
 578
 579         if (value) {
 580             value = rfc1738_escape_part(value);
 581             vstr.append("=\"", 2);
 582             vstr.append(value);
 583             vstr.append("\"", 1);
 584         }
 585
 586         hdr.clean();
 587     }
 588
 589     vary.clean();
 590 #endif
 591
 592     debugs(11, 3, "httpMakeVaryMark: " << vstr);
 593     return vstr.termedBuf();
 594 }
 595
 596 void
 597 HttpStateData::keepaliveAccounting(HttpReply *reply)
 598 {
 599     if (flags.keepalive)
 600         if (_peer)
 601             _peer->stats.n_keepalives_sent++;
 602
 603     if (reply->keep_alive) {
 604         if (_peer)
 605             _peer->stats.n_keepalives_recv++;
 606
 607         if (Config.onoff.detect_broken_server_pconns
 608                 && reply->bodySize(request->method) == -1 && !flags.chunked) {
 609             debugs(11, 1, "keepaliveAccounting: Impossible keep-alive header from '" << entry->url() << "'" );
 610             // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
 611             flags.keepalive_broken = 1;
 612         }
 613     }
 614 }
 615
 616 void
 617 HttpStateData::checkDateSkew(HttpReply *reply)
 618 {
 619     if (reply->date > -1 && !_peer) {
 620         int skew = abs((int)(reply->date - squid_curtime));
 621
 622         if (skew > 86400)
 623             debugs(11, 3, "" << request->GetHost() << "'s clock is skewed by " << skew << " seconds!");
 624     }
 625 }
 626
 627 /**
 628  * This creates the error page itself.. its likely
 629  * that the forward ported reply header max size patch
 630  * generates non http conformant error pages - in which
 631  * case the errors where should be 'BAD_GATEWAY' etc
 632  */
 633 void
 634 HttpStateData::processReplyHeader()
 635 {
 636     /** Creates a blank header. If this routine is made incremental, this will not do */
 637
 638     /* NP: all exit points to this function MUST call ctx_exit(ctx) */
 639     Ctx ctx = ctx_enter(entry->mem_obj->url);
 640
 641     debugs(11, 3, "processReplyHeader: key '" << entry->getMD5Text() << "'");
 642
 643     assert(!flags.headers_parsed);
 644
 645     if (!readBuf->hasContent()) {
 646         ctx_exit(ctx);
 647         return;
 648     }
 649
 650     http_status error = HTTP_STATUS_NONE;
 651
 652     HttpReply *newrep = new HttpReply;
 653     const bool parsed = newrep->parse(readBuf, eof, &error);
 654
 655     if (!parsed && readBuf->contentSize() > 5 && strncmp(readBuf->content(), "HTTP/", 5) != 0 && strncmp(readBuf->content(), "ICY", 3) != 0) {
 656         MemBuf *mb;
 657         HttpReply *tmprep = new HttpReply;
 658         tmprep->setHeaders(HTTP_OK, "Gatewaying", NULL, -1, -1, -1);
 659         tmprep->header.putExt("X-Transformed-From", "HTTP/0.9");
 660         mb = tmprep->pack();
 661         newrep->parse(mb, eof, &error);
 662         delete mb;
 663         delete tmprep;
 664     } else {
 665         if (!parsed && error > 0) { // unrecoverable parsing error
 666             debugs(11, 3, "processReplyHeader: Non-HTTP-compliant header: '" <<  readBuf->content() << "'");
 667             flags.headers_parsed = 1;
 668             newrep->sline.version = HttpVersion(1,1);
 669             newrep->sline.status = error;
 670             HttpReply *vrep = setVirginReply(newrep);
 671             entry->replaceHttpReply(vrep);
 672             ctx_exit(ctx);
 673             return;
 674         }
 675
 676         if (!parsed) { // need more data
 677             assert(!error);
 678             assert(!eof);
 679             delete newrep;
 680             ctx_exit(ctx);
 681             return;
 682         }
 683
 684         debugs(11, 2, "HTTP Server " << serverConnection);
 685         debugs(11, 2, "HTTP Server REPLY:\n---------\n" << readBuf->content() << "\n----------");
 686
 687         header_bytes_read = headersEnd(readBuf->content(), readBuf->contentSize());
 688         readBuf->consume(header_bytes_read);
 689     }
 690
 691     newrep->removeStaleWarnings();
 692
 693     if (newrep->sline.protocol == AnyP::PROTO_HTTP && newrep->sline.status >= 100 && newrep->sline.status < 200) {
 694         handle1xx(newrep);
 695         ctx_exit(ctx);
 696         return;
 697     }
 698
 699     flags.chunked = 0;
 700     if (newrep->sline.protocol == AnyP::PROTO_HTTP && newrep->header.chunked()) {
 701         flags.chunked = 1;
 702         httpChunkDecoder = new ChunkedCodingParser;
 703     }
 704
 705     if (!peerSupportsConnectionPinning())
 706         request->flags.connection_auth_disabled = 1;
 707
 708     HttpReply *vrep = setVirginReply(newrep);
 709     flags.headers_parsed = 1;
 710
 711     keepaliveAccounting(vrep);
 712
 713     checkDateSkew(vrep);
 714
 715     processSurrogateControl (vrep);
 716
 717     /** \todo IF the reply is a 1.0 reply, AND it has a Connection: Header
 718      * Parse the header and remove all referenced headers
 719      */
 720
 721     request->hier.peer_reply_status = newrep->sline.status;
 722
 723     ctx_exit(ctx);
 724 }
 725
 726 /// ignore or start forwarding the 1xx response (a.k.a., control message)
 727 void
 728 HttpStateData::handle1xx(HttpReply *reply)
 729 {
 730     HttpMsgPointerT<HttpReply> msg(reply); // will destroy reply if unused
 731
 732     // one 1xx at a time: we must not be called while waiting for previous 1xx
 733     Must(!flags.handling1xx);
 734     flags.handling1xx = true;
 735
 736     if (!request->canHandle1xx()) {
 737         debugs(11, 2, HERE << "ignoring client-unsupported 1xx");
 738         proceedAfter1xx();
 739         return;
 740     }
 741
 742 #if USE_HTTP_VIOLATIONS
 743     // check whether the 1xx response forwarding is allowed by squid.conf
 744     if (Config.accessList.reply) {
 745         ACLFilledChecklist ch(Config.accessList.reply, originalRequest(), NULL);
 746         ch.reply = HTTPMSGLOCK(reply);
 747         if (!ch.fastCheck()) { // TODO: support slow lookups?
 748             debugs(11, 3, HERE << "ignoring denied 1xx");
 749             proceedAfter1xx();
 750             return;
 751         }
 752     }
 753 #endif // USE_HTTP_VIOLATIONS
 754
 755     debugs(11, 2, HERE << "forwarding 1xx to client");
 756
 757     // the Sink will use this to call us back after writing 1xx to the client
 758     typedef NullaryMemFunT<HttpStateData> CbDialer;
 759     const AsyncCall::Pointer cb = JobCallback(11, 3, CbDialer, this,
 760                                   HttpStateData::proceedAfter1xx);
 761     CallJobHere1(11, 4, request->clientConnectionManager, ConnStateData,
 762                  ConnStateData::sendControlMsg, HttpControlMsg(msg, cb));
 763     // If the call is not fired, then the Sink is gone, and HttpStateData
 764     // will terminate due to an aborted store entry or another similar error.
 765     // If we get stuck, it is not handle1xx fault if we could get stuck
 766     // for similar reasons without a 1xx response.
 767 }
 768
 769 /// restores state and resumes processing after 1xx is ignored or forwarded
 770 void
 771 HttpStateData::proceedAfter1xx()
 772 {
 773     Must(flags.handling1xx);
 774
 775     debugs(11, 2, HERE << "consuming " << header_bytes_read <<
 776            " header and " << reply_bytes_read << " body bytes read after 1xx");
 777     header_bytes_read = 0;
 778     reply_bytes_read = 0;
 779
 780     CallJobHere(11, 3, this, HttpStateData, HttpStateData::processReply);
 781 }
 782
 783
 784 /**
 785  * returns true if the peer can support connection pinning
 786 */
 787 bool HttpStateData::peerSupportsConnectionPinning() const
 788 {
 789     const HttpReply *rep = entry->mem_obj->getReply();
 790     const HttpHeader *hdr = &rep->header;
 791     bool rc;
 792     String header;
 793
 794     if (!_peer)
 795         return true;
 796
 797     /*If this peer does not support connection pinning (authenticated
 798       connections) return false
 799      */
 800     if (!_peer->connection_auth)
 801         return false;
 802
 803     /*The peer supports connection pinning and the http reply status
 804       is not unauthorized, so the related connection can be pinned
 805      */
 806     if (rep->sline.status != HTTP_UNAUTHORIZED)
 807         return true;
 808
 809     /*The server respond with HTTP_UNAUTHORIZED and the peer configured
 810       with "connection-auth=on" we know that the peer supports pinned
 811       connections
 812     */
 813     if (_peer->connection_auth == 1)
 814         return true;
 815
 816     /*At this point peer has configured with "connection-auth=auto"
 817       parameter so we need some extra checks to decide if we are going
 818       to allow pinned connections or not
 819     */
 820
 821     /*if the peer configured with originserver just allow connection
 822         pinning (squid 2.6 behaviour)
 823      */
 824     if (_peer->options.originserver)
 825         return true;
 826
 827     /*if the connections it is already pinned it is OK*/
 828     if (request->flags.pinned)
 829         return true;
 830
 831     /*Allow pinned connections only if the Proxy-support header exists in
 832       reply and has in its list the "Session-Based-Authentication"
 833       which means that the peer supports connection pinning.
 834      */
 835     if (!hdr->has(HDR_PROXY_SUPPORT))
 836         return false;
 837
 838     header = hdr->getStrOrList(HDR_PROXY_SUPPORT);
 839     /* XXX This ought to be done in a case-insensitive manner */
 840     rc = (strstr(header.termedBuf(), "Session-Based-Authentication") != NULL);
 841
 842     return rc;
 843 }
 844
 845 // Called when we parsed (and possibly adapted) the headers but
 846 // had not starting storing (a.k.a., sending) the body yet.
 847 void
 848 HttpStateData::haveParsedReplyHeaders()
 849 {
 850     ServerStateData::haveParsedReplyHeaders();
 851
 852     Ctx ctx = ctx_enter(entry->mem_obj->url);
 853     HttpReply *rep = finalReply();
 854
 855     if (rep->sline.status == HTTP_PARTIAL_CONTENT &&
 856             rep->content_range)
 857         currentOffset = rep->content_range->spec.offset;
 858
 859     entry->timestampsSet();
 860
 861     /* Check if object is cacheable or not based on reply code */
 862     debugs(11, 3, "haveParsedReplyHeaders: HTTP CODE: " << rep->sline.status);
 863
 864     if (neighbors_do_private_keys)
 865         httpMaybeRemovePublic(entry, rep->sline.status);
 866
 867     if (rep->header.has(HDR_VARY)
 868 #if X_ACCELERATOR_VARY
 869             || rep->header.has(HDR_X_ACCELERATOR_VARY)
 870 #endif
 871        ) {
 872         const char *vary = httpMakeVaryMark(request, rep);
 873
 874         if (!vary) {
 875             entry->makePrivate();
 876             if (!fwd->reforwardableStatus(rep->sline.status))
 877                 EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 878             goto no_cache;
 879         }
 880
 881         entry->mem_obj->vary_headers = xstrdup(vary);
 882     }
 883
 884     /*
 885      * If its not a reply that we will re-forward, then
 886      * allow the client to get it.
 887      */
 888     if (!fwd->reforwardableStatus(rep->sline.status))
 889         EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 890
 891     switch (cacheableReply()) {
 892
 893     case 1:
 894         entry->makePublic();
 895         break;
 896
 897     case 0:
 898         entry->makePrivate();
 899         break;
 900
 901     case -1:
 902
 903 #if USE_HTTP_VIOLATIONS
 904         if (Config.negativeTtl > 0)
 905             entry->cacheNegatively();
 906         else
 907 #endif
 908             entry->makePrivate();
 909
 910         break;
 911
 912     default:
 913         assert(0);
 914
 915         break;
 916     }
 917
 918 no_cache:
 919
 920     if (!ignoreCacheControl && rep->cache_control) {
 921         if (rep->cache_control->proxyRevalidate() ||
 922                 rep->cache_control->mustRevalidate() ||
 923                 rep->cache_control->hasSMaxAge()
 924            )
 925             EBIT_SET(entry->flags, ENTRY_REVALIDATE);
 926     }
 927
 928 #if HEADERS_LOG
 929     headersLog(1, 0, request->method, rep);
 930
 931 #endif
 932
 933     ctx_exit(ctx);
 934 }
 935
 936 HttpStateData::ConnectionStatus
 937 HttpStateData::statusIfComplete() const
 938 {
 939     const HttpReply *rep = virginReply();
 940     /** \par
 941      * If the reply wants to close the connection, it takes precedence */
 942
 943     if (httpHeaderHasConnDir(&rep->header, "close"))
 944         return COMPLETE_NONPERSISTENT_MSG;
 945
 946     /** \par
 947      * If we didn't send a keep-alive request header, then this
 948      * can not be a persistent connection.
 949      */
 950     if (!flags.keepalive)
 951         return COMPLETE_NONPERSISTENT_MSG;
 952
 953     /** \par
 954      * If we haven't sent the whole request then this can not be a persistent
 955      * connection.
 956      */
 957     if (!flags.request_sent) {
 958         debugs(11, 2, "statusIfComplete: Request not yet fully sent \"" << RequestMethodStr(request->method) << " " << entry->url() << "\"" );
 959         return COMPLETE_NONPERSISTENT_MSG;
 960     }
 961
 962     /** \par
 963      * What does the reply have to say about keep-alive?
 964      */
 965     /**
 966      \bug XXX BUG?
 967      * If the origin server (HTTP/1.0) does not send a keep-alive
 968      * header, but keeps the connection open anyway, what happens?
 969      * We'll return here and http.c waits for an EOF before changing
 970      * store_status to STORE_OK.   Combine this with ENTRY_FWD_HDR_WAIT
 971      * and an error status code, and we might have to wait until
 972      * the server times out the socket.
 973      */
 974     if (!rep->keep_alive)
 975         return COMPLETE_NONPERSISTENT_MSG;
 976
 977     return COMPLETE_PERSISTENT_MSG;
 978 }
 979
 980 HttpStateData::ConnectionStatus
 981 HttpStateData::persistentConnStatus() const
 982 {
 983     debugs(11, 3, HERE << serverConnection << " eof=" << eof);
 984     if (eof) // already reached EOF
 985         return COMPLETE_NONPERSISTENT_MSG;
 986
 987     /* If server fd is closing (but we have not been notified yet), stop Comm
 988        I/O to avoid assertions. TODO: Change Comm API to handle callers that
 989        want more I/O after async closing (usually initiated by others). */
 990     // XXX: add canReceive or s/canSend/canTalkToServer/
 991     if (!Comm::IsConnOpen(serverConnection))
 992         return COMPLETE_NONPERSISTENT_MSG;
 993
 994     /** \par
 995      * In chunked response we do not know the content length but we are absolutely
 996      * sure about the end of response, so we are calling the statusIfComplete to
 997      * decide if we can be persistant
 998      */
 999     if (lastChunk && flags.chunked)
1000         return statusIfComplete();
1001
1002     const HttpReply *vrep = virginReply();
1003     debugs(11, 5, "persistentConnStatus: content_length=" << vrep->content_length);
1004
1005     const int64_t clen = vrep->bodySize(request->method);
1006
1007     debugs(11, 5, "persistentConnStatus: clen=" << clen);
1008
1009     /* If the body size is unknown we must wait for EOF */
1010     if (clen < 0)
1011         return INCOMPLETE_MSG;
1012
1013     /** \par
1014      * If the body size is known, we must wait until we've gotten all of it. */
1015     if (clen > 0) {
1016         // old technique:
1017         // if (entry->mem_obj->endOffset() < vrep->content_length + vrep->hdr_sz)
1018         const int64_t body_bytes_read = reply_bytes_read - header_bytes_read;
1019         debugs(11,5, "persistentConnStatus: body_bytes_read=" <<
1020                body_bytes_read << " content_length=" << vrep->content_length);
1021
1022         if (body_bytes_read < vrep->content_length)
1023             return INCOMPLETE_MSG;
1024
1025         if (body_bytes_truncated > 0) // already read more than needed
1026             return COMPLETE_NONPERSISTENT_MSG; // disable pconns
1027     }
1028
1029     /** \par
1030      * If there is no message body or we got it all, we can be persistent */
1031     return statusIfComplete();
1032 }
1033
1034 /*
1035  * This is the callback after some data has been read from the network
1036  */
1037 /*
1038 void
1039 HttpStateData::ReadReplyWrapper(int fd, char *buf, size_t len, comm_err_t flag, int xerrno, void *data)
1040 {
1041     HttpStateData *httpState = static_cast<HttpStateData *>(data);
1042     assert (fd == httpState->serverConnection->fd);
1043     // assert(buf == readBuf->content());
1044     PROF_start(HttpStateData_readReply);
1045     httpState->readReply(len, flag, xerrno);
1046     PROF_stop(HttpStateData_readReply);
1047 }
1048 */
1049
1050 /* XXX this function is too long! */
1051 void
1052 HttpStateData::readReply(const CommIoCbParams &io)
1053 {
1054     int bin;
1055     int clen;
1056     int len = io.size;
1057
1058     flags.do_next_read = 0;
1059
1060     debugs(11, 5, HERE << io.conn << ": len " << len << ".");
1061
1062     // Bail out early on COMM_ERR_CLOSING - close handlers will tidy up for us
1063     if (io.flag == COMM_ERR_CLOSING) {
1064         debugs(11, 3, "http socket closing");
1065         return;
1066     }
1067
1068     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1069         // TODO: should we call abortTransaction() here?
1070         return;
1071     }
1072
1073     // handle I/O errors
1074     if (io.flag != COMM_OK || len < 0) {
1075         debugs(11, 2, HERE << io.conn << ": read failure: " << xstrerror() << ".");
1076
1077         if (ignoreErrno(io.xerrno)) {
1078             flags.do_next_read = 1;
1079         } else {
1080             ErrorState *err = new ErrorState(ERR_READ_ERROR, HTTP_BAD_GATEWAY, fwd->request);
1081             err->xerrno = io.xerrno;
1082             fwd->fail(err);
1083             flags.do_next_read = 0;
1084             serverConnection->close();
1085         }
1086
1087         return;
1088     }
1089
1090     // update I/O stats
1091     if (len > 0) {
1092         readBuf->appended(len);
1093         reply_bytes_read += len;
1094 #if USE_DELAY_POOLS
1095         DelayId delayId = entry->mem_obj->mostBytesAllowed();
1096         delayId.bytesIn(len);
1097 #endif
1098
1099         kb_incr(&(statCounter.server.all.kbytes_in), len);
1100         kb_incr(&(statCounter.server.http.kbytes_in), len);
1101         IOStats.Http.reads++;
1102
1103         for (clen = len - 1, bin = 0; clen; bin++)
1104             clen >>= 1;
1105
1106         IOStats.Http.read_hist[bin]++;
1107
1108         // update peer response time stats (%<pt)
1109         const timeval &sent = request->hier.peer_http_request_sent;
1110         request->hier.peer_response_time =
1111             sent.tv_sec ? tvSubMsec(sent, current_time) : -1;
1112     }
1113
1114     /** \par
1115      * Here the RFC says we should ignore whitespace between replies, but we can't as
1116      * doing so breaks HTTP/0.9 replies beginning with witespace, and in addition
1117      * the response splitting countermeasures is extremely likely to trigger on this,
1118      * not allowing connection reuse in the first place.
1119      *
1120      * 2012-02-10: which RFC? not 2068 or 2616,
1121      *     tolerance there is all about whitespace between requests and header tokens.
1122      */
1123
1124     if (len == 0) { // reached EOF?
1125         eof = 1;
1126         flags.do_next_read = 0;
1127
1128         /* Bug 2879: Replies may terminate with \r\n then EOF instead of \r\n\r\n
1129          * Ensure here that we have at minimum two \r\n when EOF is seen.
1130          * TODO: Add eof parameter to headersEnd() and move this hack there.
1131          */
1132         if (readBuf->contentSize() && !flags.headers_parsed) {
1133             /*
1134              * Yes Henrik, there is a point to doing this.  When we
1135              * called httpProcessReplyHeader() before, we didn't find
1136              * the end of headers, but now we are definately at EOF, so
1137              * we want to process the reply headers.
1138              */
1139             /* Fake an "end-of-headers" to work around such broken servers */
1140             readBuf->append("\r\n", 2);
1141         }
1142     }
1143
1144     processReply();
1145 }
1146
1147 /// processes the already read and buffered response data, possibly after
1148 /// waiting for asynchronous 1xx control message processing
1149 void
1150 HttpStateData::processReply()
1151 {
1152
1153     if (flags.handling1xx) { // we came back after handling a 1xx response
1154         debugs(11, 5, HERE << "done with 1xx handling");
1155         flags.handling1xx = false;
1156         Must(!flags.headers_parsed);
1157     }
1158
1159     if (!flags.headers_parsed) { // have not parsed headers yet?
1160         PROF_start(HttpStateData_processReplyHeader);
1161         processReplyHeader();
1162         PROF_stop(HttpStateData_processReplyHeader);
1163
1164         if (!continueAfterParsingHeader()) // parsing error or need more data
1165             return; // TODO: send errors to ICAP
1166
1167         adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1168     }
1169
1170     // kick more reads if needed and/or process the response body, if any
1171     PROF_start(HttpStateData_processReplyBody);
1172     processReplyBody(); // may call serverComplete()
1173     PROF_stop(HttpStateData_processReplyBody);
1174 }
1175
1176 /**
1177  \retval true    if we can continue with processing the body or doing ICAP.
1178  */
1179 bool
1180 HttpStateData::continueAfterParsingHeader()
1181 {
1182     if (flags.handling1xx) {
1183         debugs(11, 5, HERE << "wait for 1xx handling");
1184         Must(!flags.headers_parsed);
1185         return false;
1186     }
1187
1188     if (!flags.headers_parsed && !eof) {
1189         debugs(11, 9, HERE << "needs more at " << readBuf->contentSize());
1190         flags.do_next_read = 1;
1191         /** \retval false If we have not finished parsing the headers and may get more data.
1192          *                Schedules more reads to retrieve the missing data.
1193          */
1194         maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1195         return false;
1196     }
1197
1198     /** If we are done with parsing, check for errors */
1199
1200     err_type error = ERR_NONE;
1201
1202     if (flags.headers_parsed) { // parsed headers, possibly with errors
1203         // check for header parsing errors
1204         if (HttpReply *vrep = virginReply()) {
1205             const http_status s = vrep->sline.status;
1206             const HttpVersion &v = vrep->sline.version;
1207             if (s == HTTP_INVALID_HEADER && v != HttpVersion(0,9)) {
1208                 debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry->url() << " AKA " << request->GetHost() << request->urlpath.termedBuf() );
1209                 error = ERR_INVALID_RESP;
1210             } else if (s == HTTP_HEADER_TOO_LARGE) {
1211                 fwd->dontRetry(true);
1212                 error = ERR_TOO_BIG;
1213             } else {
1214                 return true; // done parsing, got reply, and no error
1215             }
1216         } else {
1217             // parsed headers but got no reply
1218             debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: No reply at all for " << entry->url() << " AKA " << request->GetHost() << request->urlpath.termedBuf() );
1219             error = ERR_INVALID_RESP;
1220         }
1221     } else {
1222         assert(eof);
1223         if (readBuf->hasContent()) {
1224             error = ERR_INVALID_RESP;
1225             debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry->url() << " AKA " << request->GetHost() << request->urlpath.termedBuf() );
1226         } else {
1227             error = ERR_ZERO_SIZE_OBJECT;
1228             debugs(11, (request->flags.accelerated?DBG_IMPORTANT:2), "WARNING: HTTP: Invalid Response: No object data received for " <<
1229                    entry->url() << " AKA " << request->GetHost() << request->urlpath.termedBuf() );
1230         }
1231     }
1232
1233     assert(error != ERR_NONE);
1234     entry->reset();
1235     fwd->fail(new ErrorState(error, HTTP_BAD_GATEWAY, fwd->request));
1236     flags.do_next_read = 0;
1237     serverConnection->close();
1238     return false; // quit on error
1239 }
1240
1241 /** truncate what we read if we read too much so that writeReplyBody()
1242     writes no more than what we should have read */
1243 void
1244 HttpStateData::truncateVirginBody()
1245 {
1246     assert(flags.headers_parsed);
1247
1248     HttpReply *vrep = virginReply();
1249     int64_t clen = -1;
1250     if (!vrep->expectingBody(request->method, clen) || clen < 0)
1251         return; // no body or a body of unknown size, including chunked
1252
1253     const int64_t body_bytes_read = reply_bytes_read - header_bytes_read;
1254     if (body_bytes_read - body_bytes_truncated <= clen)
1255         return; // we did not read too much or already took care of the extras
1256
1257     if (const int64_t extras = body_bytes_read - body_bytes_truncated - clen) {
1258         // server sent more that the advertised content length
1259         debugs(11,5, HERE << "body_bytes_read=" << body_bytes_read <<
1260                " clen=" << clen << '/' << vrep->content_length <<
1261                " body_bytes_truncated=" << body_bytes_truncated << '+' << extras);
1262
1263         readBuf->truncate(extras);
1264         body_bytes_truncated += extras;
1265     }
1266 }
1267
1268 /**
1269  * Call this when there is data from the origin server
1270  * which should be sent to either StoreEntry, or to ICAP...
1271  */
1272 void
1273 HttpStateData::writeReplyBody()
1274 {
1275     truncateVirginBody(); // if needed
1276     const char *data = readBuf->content();
1277     int len = readBuf->contentSize();
1278     addVirginReplyBody(data, len);
1279     readBuf->consume(len);
1280 }
1281
1282 bool
1283 HttpStateData::decodeAndWriteReplyBody()
1284 {
1285     const char *data = NULL;
1286     int len;
1287     bool wasThereAnException = false;
1288     assert(flags.chunked);
1289     assert(httpChunkDecoder);
1290     SQUID_ENTER_THROWING_CODE();
1291     MemBuf decodedData;
1292     decodedData.init();
1293     const bool doneParsing = httpChunkDecoder->parse(readBuf,&decodedData);
1294     len = decodedData.contentSize();
1295     data=decodedData.content();
1296     addVirginReplyBody(data, len);
1297     if (doneParsing) {
1298         lastChunk = 1;
1299         flags.do_next_read = 0;
1300     }
1301     SQUID_EXIT_THROWING_CODE(wasThereAnException);
1302     return wasThereAnException;
1303 }
1304
1305 /**
1306  * processReplyBody has two purposes:
1307  *  1 - take the reply body data, if any, and put it into either
1308  *      the StoreEntry, or give it over to ICAP.
1309  *  2 - see if we made it to the end of the response (persistent
1310  *      connections and such)
1311  */
1312 void
1313 HttpStateData::processReplyBody()
1314 {
1315     Ip::Address client_addr;
1316     bool ispinned = false;
1317
1318     if (!flags.headers_parsed) {
1319         flags.do_next_read = 1;
1320         maybeReadVirginBody();
1321         return;
1322     }
1323
1324 #if USE_ADAPTATION
1325     debugs(11,5, HERE << "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
1326     if (adaptationAccessCheckPending)
1327         return;
1328
1329 #endif
1330
1331     /*
1332      * At this point the reply headers have been parsed and consumed.
1333      * That means header content has been removed from readBuf and
1334      * it contains only body data.
1335      */
1336     if (entry->isAccepting()) {
1337         if (flags.chunked) {
1338             if (!decodeAndWriteReplyBody()) {
1339                 flags.do_next_read = 0;
1340                 serverComplete();
1341                 return;
1342             }
1343         } else
1344             writeReplyBody();
1345     }
1346
1347     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1348         /*
1349          * The above writeReplyBody() call could ABORT this entry,
1350          * in that case, the server FD should already be closed.
1351          * there's nothing for us to do.
1352          */
1353         (void) 0;
1354     } else
1355         switch (persistentConnStatus()) {
1356         case INCOMPLETE_MSG: {
1357             debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection);
1358             /* Wait for more data or EOF condition */
1359             AsyncCall::Pointer nil;
1360             if (flags.keepalive_broken) {
1361                 commSetConnTimeout(serverConnection, 10, nil);
1362             } else {
1363                 commSetConnTimeout(serverConnection, Config.Timeout.read, nil);
1364             }
1365
1366             flags.do_next_read = 1;
1367         }
1368         break;
1369
1370         case COMPLETE_PERSISTENT_MSG:
1371             debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection);
1372             /* yes we have to clear all these! */
1373             commUnsetConnTimeout(serverConnection);
1374             flags.do_next_read = 0;
1375
1376             comm_remove_close_handler(serverConnection->fd, closeHandler);
1377             closeHandler = NULL;
1378             fwd->unregister(serverConnection);
1379
1380             if (request->flags.spoof_client_ip)
1381                 client_addr = request->client_addr;
1382
1383
1384             if (request->flags.pinned) {
1385                 ispinned = true;
1386             } else if (request->flags.connection_auth && request->flags.auth_sent) {
1387                 ispinned = true;
1388             }
1389
1390             if (request->pinnedConnection() && ispinned) {
1391                 request->pinnedConnection()->pinConnection(serverConnection, request, _peer,
1392                         (request->flags.connection_auth != 0));
1393             } else {
1394                 fwd->pconnPush(serverConnection, request->peer_host ? request->peer_host : request->GetHost());
1395             }
1396
1397             serverConnection = NULL;
1398             serverComplete();
1399             return;
1400
1401         case COMPLETE_NONPERSISTENT_MSG:
1402             debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection);
1403             serverComplete();
1404             return;
1405         }
1406
1407     maybeReadVirginBody();
1408 }
1409
1410 void
1411 HttpStateData::maybeReadVirginBody()
1412 {
1413     // too late to read
1414     if (!Comm::IsConnOpen(serverConnection) || fd_table[serverConnection->fd].closing())
1415         return;
1416
1417     // we may need to grow the buffer if headers do not fit
1418     const int minRead = flags.headers_parsed ? 0 :1024;
1419     const int read_size = replyBodySpace(*readBuf, minRead);
1420
1421     debugs(11,9, HERE << (flags.do_next_read ? "may" : "wont") <<
1422            " read up to " << read_size << " bytes from " << serverConnection);
1423
1424     /*
1425      * why <2? Because delayAwareRead() won't actually read if
1426      * you ask it to read 1 byte.  The delayed read request
1427      * just gets re-queued until the client side drains, then
1428      * the I/O thread hangs.  Better to not register any read
1429      * handler until we get a notification from someone that
1430      * its okay to read again.
1431      */
1432     if (read_size < 2)
1433         return;
1434
1435     if (flags.do_next_read) {
1436         flags.do_next_read = 0;
1437         typedef CommCbMemFunT<HttpStateData, CommIoCbParams> Dialer;
1438         entry->delayAwareRead(serverConnection, readBuf->space(read_size), read_size,
1439                               JobCallback(11, 5, Dialer, this,  HttpStateData::readReply));
1440     }
1441 }
1442
1443 /// called after writing the very last request byte (body, last-chunk, etc)
1444 void
1445 HttpStateData::wroteLast(const CommIoCbParams &io)
1446 {
1447     debugs(11, 5, HERE << serverConnection << ": size " << io.size << ": errflag " << io.flag << ".");
1448 #if URL_CHECKSUM_DEBUG
1449
1450     entry->mem_obj->checkUrlChecksum();
1451 #endif
1452
1453     if (io.size > 0) {
1454         fd_bytes(io.fd, io.size, FD_WRITE);
1455         kb_incr(&(statCounter.server.all.kbytes_out), io.size);
1456         kb_incr(&(statCounter.server.http.kbytes_out), io.size);
1457     }
1458
1459     if (io.flag == COMM_ERR_CLOSING)
1460         return;
1461
1462     if (io.flag) {
1463         ErrorState *err = new ErrorState(ERR_WRITE_ERROR, HTTP_BAD_GATEWAY, fwd->request);
1464         err->xerrno = io.xerrno;
1465         fwd->fail(err);
1466         serverConnection->close();
1467         return;
1468     }
1469
1470     sendComplete();
1471 }
1472
1473 /// successfully wrote the entire request (including body, last-chunk, etc.)
1474 void
1475 HttpStateData::sendComplete()
1476 {
1477     /*
1478      * Set the read timeout here because it hasn't been set yet.
1479      * We only set the read timeout after the request has been
1480      * fully written to the server-side.  If we start the timeout
1481      * after connection establishment, then we are likely to hit
1482      * the timeout for POST/PUT requests that have very large
1483      * request bodies.
1484      */
1485     typedef CommCbMemFunT<HttpStateData, CommTimeoutCbParams> TimeoutDialer;
1486     AsyncCall::Pointer timeoutCall =  JobCallback(11, 5,
1487                                       TimeoutDialer, this, HttpStateData::httpTimeout);
1488
1489     commSetConnTimeout(serverConnection, Config.Timeout.read, timeoutCall);
1490
1491     flags.request_sent = 1;
1492
1493     request->hier.peer_http_request_sent = current_time;
1494 }
1495
1496 // Close the HTTP server connection. Used by serverComplete().
1497 void
1498 HttpStateData::closeServer()
1499 {
1500     debugs(11,5, HERE << "closing HTTP server " << serverConnection << " this " << this);
1501
1502     if (Comm::IsConnOpen(serverConnection)) {
1503         fwd->unregister(serverConnection);
1504         comm_remove_close_handler(serverConnection->fd, closeHandler);
1505         closeHandler = NULL;
1506         serverConnection->close();
1507     }
1508 }
1509
1510 bool
1511 HttpStateData::doneWithServer() const
1512 {
1513     return !Comm::IsConnOpen(serverConnection);
1514 }
1515
1516 /*
1517  * Fixup authentication request headers for special cases
1518  */
1519 static void
1520 httpFixupAuthentication(HttpRequest * request, const HttpHeader * hdr_in, HttpHeader * hdr_out, http_state_flags flags)
1521 {
1522     http_hdr_type header = flags.originpeer ? HDR_AUTHORIZATION : HDR_PROXY_AUTHORIZATION;
1523
1524     /* Nothing to do unless we are forwarding to a peer */
1525     if (!request->flags.proxying)
1526         return;
1527
1528     /* Needs to be explicitly enabled */
1529     if (!request->peer_login)
1530         return;
1531
1532     /* Maybe already dealt with? */
1533     if (hdr_out->has(header))
1534         return;
1535
1536     /* Nothing to do here for PASSTHRU */
1537     if (strcmp(request->peer_login, "PASSTHRU") == 0)
1538         return;
1539
1540     /* PROXYPASS is a special case, single-signon to servers with the proxy password (basic only) */
1541     if (flags.originpeer && strcmp(request->peer_login, "PROXYPASS") == 0 && hdr_in->has(HDR_PROXY_AUTHORIZATION)) {
1542         const char *auth = hdr_in->getStr(HDR_PROXY_AUTHORIZATION);
1543
1544         if (auth && strncasecmp(auth, "basic ", 6) == 0) {
1545             hdr_out->putStr(header, auth);
1546             return;
1547         }
1548     }
1549
1550     /* Special mode to pass the username to the upstream cache */
1551     if (*request->peer_login == '*') {
1552         char loginbuf[256];
1553         const char *username = "-";
1554
1555         if (request->extacl_user.size())
1556             username = request->extacl_user.termedBuf();
1557 #if USE_AUTH
1558         else if (request->auth_user_request != NULL)
1559             username = request->auth_user_request->username();
1560 #endif
1561
1562         snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, request->peer_login + 1);
1563
1564         httpHeaderPutStrf(hdr_out, header, "Basic %s",
1565                           old_base64_encode(loginbuf));
1566         return;
1567     }
1568
1569     /* external_acl provided credentials */
1570     if (request->extacl_user.size() && request->extacl_passwd.size() &&
1571             (strcmp(request->peer_login, "PASS") == 0 ||
1572              strcmp(request->peer_login, "PROXYPASS") == 0)) {
1573         char loginbuf[256];
1574         snprintf(loginbuf, sizeof(loginbuf), SQUIDSTRINGPH ":" SQUIDSTRINGPH,
1575                  SQUIDSTRINGPRINT(request->extacl_user),
1576                  SQUIDSTRINGPRINT(request->extacl_passwd));
1577         httpHeaderPutStrf(hdr_out, header, "Basic %s",
1578                           old_base64_encode(loginbuf));
1579         return;
1580     }
1581
1582     /* Kerberos login to peer */
1583 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1584     if (strncmp(request->peer_login, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1585         char *Token=NULL;
1586         char *PrincipalName=NULL,*p;
1587         if ((p=strchr(request->peer_login,':')) != NULL ) {
1588             PrincipalName=++p;
1589         }
1590         Token = peer_proxy_negotiate_auth(PrincipalName, request->peer_host);
1591         if (Token) {
1592             httpHeaderPutStrf(hdr_out, header, "Negotiate %s",Token);
1593         }
1594         return;
1595     }
1596 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1597
1598     httpHeaderPutStrf(hdr_out, header, "Basic %s",
1599                       old_base64_encode(request->peer_login));
1600     return;
1601 }
1602
1603 /*
1604  * build request headers and append them to a given MemBuf
1605  * used by buildRequestPrefix()
1606  * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1607  */
1608 void
1609 HttpStateData::httpBuildRequestHeader(HttpRequest * request,
1610                                       StoreEntry * entry,
1611                                       HttpHeader * hdr_out,
1612                                       const http_state_flags flags)
1613 {
1614     /* building buffer for complex strings */
1615 #define BBUF_SZ (MAX_URL+32)
1616     LOCAL_ARRAY(char, bbuf, BBUF_SZ);
1617     LOCAL_ARRAY(char, ntoabuf, MAX_IPSTRLEN);
1618     const HttpHeader *hdr_in = &request->header;
1619     const HttpHeaderEntry *e = NULL;
1620     HttpHeaderPos pos = HttpHeaderInitPos;
1621     assert (hdr_out->owner == hoRequest);
1622
1623     /* append our IMS header */
1624     if (request->lastmod > -1)
1625         hdr_out->putTime(HDR_IF_MODIFIED_SINCE, request->lastmod);
1626
1627     bool we_do_ranges = decideIfWeDoRanges (request);
1628
1629     String strConnection (hdr_in->getList(HDR_CONNECTION));
1630
1631     while ((e = hdr_in->getEntry(&pos)))
1632         copyOneHeaderFromClientsideRequestToUpstreamRequest(e, strConnection, request, hdr_out, we_do_ranges, flags);
1633
1634     /* Abstraction break: We should interpret multipart/byterange responses
1635      * into offset-length data, and this works around our inability to do so.
1636      */
1637     if (!we_do_ranges && request->multipartRangeRequest()) {
1638         /* don't cache the result */
1639         request->flags.cachable = 0;
1640         /* pretend it's not a range request */
1641         delete request->range;
1642         request->range = NULL;
1643         request->flags.range = 0;
1644     }
1645
1646     /* append Via */
1647     if (Config.onoff.via) {
1648         String strVia;
1649         strVia = hdr_in->getList(HDR_VIA);
1650         snprintf(bbuf, BBUF_SZ, "%d.%d %s",
1651                  request->http_ver.major,
1652                  request->http_ver.minor, ThisCache);
1653         strListAdd(&strVia, bbuf, ',');
1654         hdr_out->putStr(HDR_VIA, strVia.termedBuf());
1655         strVia.clean();
1656     }
1657
1658     if (request->flags.accelerated) {
1659         /* Append Surrogate-Capabilities */
1660         String strSurrogate(hdr_in->getList(HDR_SURROGATE_CAPABILITY));
1661 #if USE_SQUID_ESI
1662         snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0 ESI/1.0\"", Config.Accel.surrogate_id);
1663 #else
1664         snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0\"", Config.Accel.surrogate_id);
1665 #endif
1666         strListAdd(&strSurrogate, bbuf, ',');
1667         hdr_out->putStr(HDR_SURROGATE_CAPABILITY, strSurrogate.termedBuf());
1668     }
1669
1670     /** \pre Handle X-Forwarded-For */
1671     if (strcmp(opt_forwarded_for, "delete") != 0) {
1672
1673         String strFwd = hdr_in->getList(HDR_X_FORWARDED_FOR);
1674
1675         if (strFwd.size() > 65536/2) {
1676             // There is probably a forwarding loop with Via detection disabled.
1677             // If we do nothing, String will assert on overflow soon.
1678             // TODO: Terminate all transactions with huge XFF?
1679             strFwd = "error";
1680
1681             static int warnedCount = 0;
1682             if (warnedCount++ < 100) {
1683                 const char *url = entry ? entry->url() : urlCanonical(request);
1684                 debugs(11, 1, "Warning: likely forwarding loop with " << url);
1685             }
1686         }
1687
1688         if (strcmp(opt_forwarded_for, "on") == 0) {
1689             /** If set to ON - append client IP or 'unknown'. */
1690             if ( request->client_addr.IsNoAddr() )
1691                 strListAdd(&strFwd, "unknown", ',');
1692             else
1693                 strListAdd(&strFwd, request->client_addr.NtoA(ntoabuf, MAX_IPSTRLEN), ',');
1694         } else if (strcmp(opt_forwarded_for, "off") == 0) {
1695             /** If set to OFF - append 'unknown'. */
1696             strListAdd(&strFwd, "unknown", ',');
1697         } else if (strcmp(opt_forwarded_for, "transparent") == 0) {
1698             /** If set to TRANSPARENT - pass through unchanged. */
1699         } else if (strcmp(opt_forwarded_for, "truncate") == 0) {
1700             /** If set to TRUNCATE - drop existing list and replace with client IP or 'unknown'. */
1701             if ( request->client_addr.IsNoAddr() )
1702                 strFwd = "unknown";
1703             else
1704                 strFwd = request->client_addr.NtoA(ntoabuf, MAX_IPSTRLEN);
1705         }
1706         if (strFwd.size() > 0)
1707             hdr_out->putStr(HDR_X_FORWARDED_FOR, strFwd.termedBuf());
1708     }
1709     /** If set to DELETE - do not copy through. */
1710
1711     /* append Host if not there already */
1712     if (!hdr_out->has(HDR_HOST)) {
1713         if (request->peer_domain) {
1714             hdr_out->putStr(HDR_HOST, request->peer_domain);
1715         } else if (request->port == urlDefaultPort(request->protocol)) {
1716             /* use port# only if not default */
1717             hdr_out->putStr(HDR_HOST, request->GetHost());
1718         } else {
1719             httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d",
1720                               request->GetHost(),
1721                               (int) request->port);
1722         }
1723     }
1724
1725     /* append Authorization if known in URL, not in header and going direct */
1726     if (!hdr_out->has(HDR_AUTHORIZATION)) {
1727         if (!request->flags.proxying && request->login && *request->login) {
1728             httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s",
1729                               old_base64_encode(request->login));
1730         }
1731     }
1732
1733     /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
1734     httpFixupAuthentication(request, hdr_in, hdr_out, flags);
1735
1736     /* append Cache-Control, add max-age if not there already */
1737     {
1738         HttpHdrCc *cc = hdr_in->getCc();
1739
1740         if (!cc)
1741             cc = new HttpHdrCc();
1742
1743 #if 0 /* see bug 2330 */
1744         /* Set no-cache if determined needed but not found */
1745         if (request->flags.nocache)
1746             EBIT_SET(cc->mask, CC_NO_CACHE);
1747 #endif
1748
1749         /* Add max-age only without no-cache */
1750         if (!cc->hasMaxAge() && !cc->noCache()) {
1751             const char *url =
1752                 entry ? entry->url() : urlCanonical(request);
1753             cc->maxAge(getMaxAge(url));
1754
1755         }
1756
1757         /* Enforce sibling relations */
1758         if (flags.only_if_cached)
1759             cc->onlyIfCached(true);
1760
1761         hdr_out->putCc(cc);
1762
1763         delete cc;
1764     }
1765
1766     /* maybe append Connection: keep-alive */
1767     if (flags.keepalive) {
1768         hdr_out->putStr(HDR_CONNECTION, "keep-alive");
1769     }
1770
1771     /* append Front-End-Https */
1772     if (flags.front_end_https) {
1773         if (flags.front_end_https == 1 || request->protocol == AnyP::PROTO_HTTPS)
1774             hdr_out->putStr(HDR_FRONT_END_HTTPS, "On");
1775     }
1776
1777     if (flags.chunked_request) {
1778         // Do not just copy the original value so that if the client-side
1779         // starts decode other encodings, this code may remain valid.
1780         hdr_out->putStr(HDR_TRANSFER_ENCODING, "chunked");
1781     }
1782
1783     /* Now mangle the headers. */
1784     if (Config2.onoff.mangle_request_headers)
1785         httpHdrMangleList(hdr_out, request, ROR_REQUEST);
1786
1787     strConnection.clean();
1788 }
1789
1790 /**
1791  * Decides whether a particular header may be cloned from the received Clients request
1792  * to our outgoing fetch request.
1793  */
1794 void
1795 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request, HttpHeader * hdr_out, const int we_do_ranges, const http_state_flags flags)
1796 {
1797     debugs(11, 5, "httpBuildRequestHeader: " << e->name << ": " << e->value );
1798
1799     switch (e->id) {
1800
1801         /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid should not pass on. */
1802
1803     case HDR_PROXY_AUTHORIZATION:
1804         /** \par Proxy-Authorization:
1805          * Only pass on proxy authentication to peers for which
1806          * authentication forwarding is explicitly enabled
1807          */
1808         if (!flags.originpeer && flags.proxying && request->peer_login &&
1809                 (strcmp(request->peer_login, "PASS") == 0 ||
1810                  strcmp(request->peer_login, "PROXYPASS") == 0 ||
1811                  strcmp(request->peer_login, "PASSTHRU") == 0)) {
1812             hdr_out->addEntry(e->clone());
1813         }
1814         break;
1815
1816         /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid does not pass on. */
1817
1818     case HDR_CONNECTION:          /** \par Connection: */
1819     case HDR_TE:                  /** \par TE: */
1820     case HDR_KEEP_ALIVE:          /** \par Keep-Alive: */
1821     case HDR_PROXY_AUTHENTICATE:  /** \par Proxy-Authenticate: */
1822     case HDR_TRAILER:             /** \par Trailer: */
1823     case HDR_UPGRADE:             /** \par Upgrade: */
1824     case HDR_TRANSFER_ENCODING:   /** \par Transfer-Encoding: */
1825         break;
1826
1827
1828         /** \par OTHER headers I haven't bothered to track down yet. */
1829
1830     case HDR_AUTHORIZATION:
1831         /** \par WWW-Authorization:
1832          * Pass on WWW authentication */
1833
1834         if (!flags.originpeer) {
1835             hdr_out->addEntry(e->clone());
1836         } else {
1837             /** \note In accelerators, only forward authentication if enabled
1838              * (see also httpFixupAuthentication for special cases)
1839              */
1840             if (request->peer_login &&
1841                     (strcmp(request->peer_login, "PASS") == 0 ||
1842                      strcmp(request->peer_login, "PASSTHRU") == 0 ||
1843                      strcmp(request->peer_login, "PROXYPASS") == 0)) {
1844                 hdr_out->addEntry(e->clone());
1845             }
1846         }
1847
1848         break;
1849
1850     case HDR_HOST:
1851         /** \par Host:
1852          * Normally Squid rewrites the Host: header.
1853          * However, there is one case when we don't: If the URL
1854          * went through our redirector and the admin configured
1855          * 'redir_rewrites_host' to be off.
1856          */
1857         if (request->peer_domain)
1858             hdr_out->putStr(HDR_HOST, request->peer_domain);
1859         else if (request->flags.redirected && !Config.onoff.redir_rewrites_host)
1860             hdr_out->addEntry(e->clone());
1861         else {
1862             /* use port# only if not default */
1863
1864             if (request->port == urlDefaultPort(request->protocol)) {
1865                 hdr_out->putStr(HDR_HOST, request->GetHost());
1866             } else {
1867                 httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d",
1868                                   request->GetHost(),
1869                                   (int) request->port);
1870             }
1871         }
1872
1873         break;
1874
1875     case HDR_IF_MODIFIED_SINCE:
1876         /** \par If-Modified-Since:
1877         * append unless we added our own;
1878          * \note at most one client's ims header can pass through */
1879
1880         if (!hdr_out->has(HDR_IF_MODIFIED_SINCE))
1881             hdr_out->addEntry(e->clone());
1882
1883         break;
1884
1885     case HDR_MAX_FORWARDS:
1886         /** \par Max-Forwards:
1887          * pass only on TRACE or OPTIONS requests */
1888         if (request->method == METHOD_TRACE || request->method == METHOD_OPTIONS) {
1889             const int64_t hops = e->getInt64();
1890
1891             if (hops > 0)
1892                 hdr_out->putInt64(HDR_MAX_FORWARDS, hops - 1);
1893         }
1894
1895         break;
1896
1897     case HDR_VIA:
1898         /** \par Via:
1899          * If Via is disabled then forward any received header as-is.
1900          * Otherwise leave for explicit updated addition later. */
1901
1902         if (!Config.onoff.via)
1903             hdr_out->addEntry(e->clone());
1904
1905         break;
1906
1907     case HDR_RANGE:
1908
1909     case HDR_IF_RANGE:
1910
1911     case HDR_REQUEST_RANGE:
1912         /** \par Range:, If-Range:, Request-Range:
1913          * Only pass if we accept ranges */
1914         if (!we_do_ranges)
1915             hdr_out->addEntry(e->clone());
1916
1917         break;
1918
1919     case HDR_PROXY_CONNECTION: // SHOULD ignore. But doing so breaks things.
1920         break;
1921
1922     case HDR_CONTENT_LENGTH:
1923         // pass through unless we chunk; also, keeping this away from default
1924         // prevents request smuggling via Connection: Content-Length tricks
1925         if (!flags.chunked_request)
1926             hdr_out->addEntry(e->clone());
1927         break;
1928
1929     case HDR_X_FORWARDED_FOR:
1930
1931     case HDR_CACHE_CONTROL:
1932         /** \par X-Forwarded-For:, Cache-Control:
1933          * handled specially by Squid, so leave off for now.
1934          * append these after the loop if needed */
1935         break;
1936
1937     case HDR_FRONT_END_HTTPS:
1938         /** \par Front-End-Https:
1939          * Pass thru only if peer is configured with front-end-https */
1940         if (!flags.front_end_https)
1941             hdr_out->addEntry(e->clone());
1942
1943         break;
1944
1945     default:
1946         /** \par default.
1947          * pass on all other header fields
1948          * which are NOT listed by the special Connection: header. */
1949
1950         if (strConnection.size()>0 && strListIsMember(&strConnection, e->name.termedBuf(), ',')) {
1951             debugs(11, 2, "'" << e->name << "' header cropped by Connection: definition");
1952             return;
1953         }
1954
1955         hdr_out->addEntry(e->clone());
1956     }
1957 }
1958
1959 bool
1960 HttpStateData::decideIfWeDoRanges (HttpRequest * request)
1961 {
1962     bool result = true;
1963     /* decide if we want to do Ranges ourselves
1964      * and fetch the whole object now)
1965      * We want to handle Ranges ourselves iff
1966      *    - we can actually parse client Range specs
1967      *    - the specs are expected to be simple enough (e.g. no out-of-order ranges)
1968      *    - reply will be cachable
1969      * (If the reply will be uncachable we have to throw it away after
1970      *  serving this request, so it is better to forward ranges to
1971      *  the server and fetch only the requested content)
1972      */
1973
1974     int64_t roffLimit = request->getRangeOffsetLimit();
1975
1976     if (NULL == request->range || !request->flags.cachable
1977             || request->range->offsetLimitExceeded(roffLimit) || request->flags.connection_auth)
1978         result = false;
1979
1980     debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
1981            request->range << ", cachable: " <<
1982            request->flags.cachable << "; we_do_ranges: " << result);
1983
1984     return result;
1985 }
1986
1987 /* build request prefix and append it to a given MemBuf;
1988  * return the length of the prefix */
1989 mb_size_t
1990 HttpStateData::buildRequestPrefix(MemBuf * mb)
1991 {
1992     const int offset = mb->size;
1993     HttpVersion httpver(1,1);
1994     const char * url;
1995     if (_peer && !_peer->options.originserver)
1996         url = entry->url();
1997     else
1998         url = request->urlpath.termedBuf();
1999     mb->Printf("%s %s %s/%d.%d\r\n",
2000                RequestMethodStr(request->method),
2001                url && *url ? url : "/",
2002                AnyP::ProtocolType_str[httpver.protocol],
2003                httpver.major,httpver.minor);
2004     /* build and pack headers */
2005     {
2006         HttpHeader hdr(hoRequest);
2007         Packer p;
2008         httpBuildRequestHeader(request, entry, &hdr, flags);
2009
2010         if (request->flags.pinned && request->flags.connection_auth)
2011             request->flags.auth_sent = 1;
2012         else if (hdr.has(HDR_AUTHORIZATION))
2013             request->flags.auth_sent = 1;
2014
2015         packerToMemInit(&p, mb);
2016         hdr.packInto(&p);
2017         hdr.clean();
2018         packerClean(&p);
2019     }
2020     /* append header terminator */
2021     mb->append(crlf, 2);
2022     return mb->size - offset;
2023 }
2024
2025 /* This will be called when connect completes. Write request. */
2026 bool
2027 HttpStateData::sendRequest()
2028 {
2029     MemBuf mb;
2030
2031     debugs(11, 5, HERE << serverConnection << ", request " << request << ", this " << this << ".");
2032
2033     if (!Comm::IsConnOpen(serverConnection)) {
2034         debugs(11,3, HERE << "cannot send request to closing " << serverConnection);
2035         assert(closeHandler != NULL);
2036         return false;
2037     }
2038
2039     typedef CommCbMemFunT<HttpStateData, CommTimeoutCbParams> TimeoutDialer;
2040     AsyncCall::Pointer timeoutCall =  JobCallback(11, 5,
2041                                       TimeoutDialer, this, HttpStateData::httpTimeout);
2042     commSetConnTimeout(serverConnection, Config.Timeout.lifetime, timeoutCall);
2043     flags.do_next_read = 1;
2044     maybeReadVirginBody();
2045
2046     if (request->body_pipe != NULL) {
2047         if (!startRequestBodyFlow()) // register to receive body data
2048             return false;
2049         typedef CommCbMemFunT<HttpStateData, CommIoCbParams> Dialer;
2050         requestSender = JobCallback(11,5,
2051                                     Dialer, this, HttpStateData::sentRequestBody);
2052
2053         Must(!flags.chunked_request);
2054         // use chunked encoding if we do not know the length
2055         if (request->content_length < 0)
2056             flags.chunked_request = 1;
2057     } else {
2058         assert(!requestBodySource);
2059         typedef CommCbMemFunT<HttpStateData, CommIoCbParams> Dialer;
2060         requestSender = JobCallback(11,5,
2061                                     Dialer, this,  HttpStateData::wroteLast);
2062     }
2063
2064     if (_peer != NULL) {
2065         if (_peer->options.originserver) {
2066             flags.proxying = 0;
2067             flags.originpeer = 1;
2068         } else {
2069             flags.proxying = 1;
2070             flags.originpeer = 0;
2071         }
2072     } else {
2073         flags.proxying = 0;
2074         flags.originpeer = 0;
2075     }
2076
2077     /*
2078      * Is keep-alive okay for all request methods?
2079      */
2080     if (request->flags.must_keepalive)
2081         flags.keepalive = 1;
2082     else if (!Config.onoff.server_pconns)
2083         flags.keepalive = 0;
2084     else if (_peer == NULL)
2085         flags.keepalive = 1;
2086     else if (_peer->stats.n_keepalives_sent < 10)
2087         flags.keepalive = 1;
2088     else if ((double) _peer->stats.n_keepalives_recv /
2089              (double) _peer->stats.n_keepalives_sent > 0.50)
2090         flags.keepalive = 1;
2091
2092     if (_peer) {
2093         /*The old code here was
2094           if (neighborType(_peer, request) == PEER_SIBLING && ...
2095           which is equivalent to:
2096           if (neighborType(_peer, NULL) == PEER_SIBLING && ...
2097           or better:
2098           if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2099                  _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2100                flags.only_if_cached = 1;
2101
2102            But I suppose it was a bug
2103          */
2104         if (neighborType(_peer, request) == PEER_SIBLING &&
2105                 !_peer->options.allow_miss)
2106             flags.only_if_cached = 1;
2107
2108         flags.front_end_https = _peer->front_end_https;
2109     }
2110
2111     mb.init();
2112     request->peer_host=_peer?_peer->host:NULL;
2113     buildRequestPrefix(&mb);
2114
2115     debugs(11, 2, "HTTP Server " << serverConnection);
2116     debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb.buf << "\n----------");
2117
2118     Comm::Write(serverConnection, &mb, requestSender);
2119     return true;
2120 }
2121
2122 bool
2123 HttpStateData::getMoreRequestBody(MemBuf &buf)
2124 {
2125     // parent's implementation can handle the no-encoding case
2126     if (!flags.chunked_request)
2127         return ServerStateData::getMoreRequestBody(buf);
2128
2129     MemBuf raw;
2130
2131     Must(requestBodySource != NULL);
2132     if (!requestBodySource->getMoreData(raw))
2133         return false; // no request body bytes to chunk yet
2134
2135     // optimization: pre-allocate buffer size that should be enough
2136     const mb_size_t rawDataSize = raw.contentSize();
2137     // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2138     buf.init(16 + 2 + rawDataSize + 2 + 5, raw.max_capacity);
2139
2140     buf.Printf("%x\r\n", static_cast<unsigned int>(rawDataSize));
2141     buf.append(raw.content(), rawDataSize);
2142     buf.Printf("\r\n");
2143
2144     Must(rawDataSize > 0); // we did not accidently created last-chunk above
2145
2146     // Do not send last-chunk unless we successfully received everything
2147     if (receivedWholeRequestBody) {
2148         Must(!flags.sentLastChunk);
2149         flags.sentLastChunk = true;
2150         buf.append("0\r\n\r\n", 5);
2151     }
2152
2153     return true;
2154 }
2155
2156 void
2157 httpStart(FwdState *fwd)
2158 {
2159     debugs(11, 3, "httpStart: \"" << RequestMethodStr(fwd->request->method) << " " << fwd->entry->url() << "\"" );
2160     AsyncJob::Start(new HttpStateData(fwd));
2161 }
2162
2163 void
2164 HttpStateData::start()
2165 {
2166     if (!sendRequest()) {
2167         debugs(11, 3, "httpStart: aborted");
2168         mustStop("HttpStateData::start failed");
2169         return;
2170     }
2171
2172     statCounter.server.all.requests++;
2173     statCounter.server.http.requests++;
2174
2175     /*
2176      * We used to set the read timeout here, but not any more.
2177      * Now its set in httpSendComplete() after the full request,
2178      * including request body, has been written to the server.
2179      */
2180 }
2181
2182 /// if broken posts are enabled for the request, try to fix and return true
2183 bool
2184 HttpStateData::finishingBrokenPost()
2185 {
2186 #if USE_HTTP_VIOLATIONS
2187     if (!Config.accessList.brokenPosts) {
2188         debugs(11, 5, HERE << "No brokenPosts list");
2189         return false;
2190     }
2191
2192     ACLFilledChecklist ch(Config.accessList.brokenPosts, originalRequest(), NULL);
2193     if (!ch.fastCheck()) {
2194         debugs(11, 5, HERE << "didn't match brokenPosts");
2195         return false;
2196     }
2197
2198     if (!Comm::IsConnOpen(serverConnection)) {
2199         debugs(11, 3, HERE << "ignoring broken POST for closed " << serverConnection);
2200         assert(closeHandler != NULL);
2201         return true; // prevent caller from proceeding as if nothing happened
2202     }
2203
2204     debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2205     typedef CommCbMemFunT<HttpStateData, CommIoCbParams> Dialer;
2206     requestSender = JobCallback(11,5,
2207                                 Dialer, this, HttpStateData::wroteLast);
2208     Comm::Write(serverConnection, "\r\n", 2, requestSender, NULL);
2209     return true;
2210 #else
2211     return false;
2212 #endif /* USE_HTTP_VIOLATIONS */
2213 }
2214
2215 /// if needed, write last-chunk to end the request body and return true
2216 bool
2217 HttpStateData::finishingChunkedRequest()
2218 {
2219     if (flags.sentLastChunk) {
2220         debugs(11, 5, HERE << "already sent last-chunk");
2221         return false;
2222     }
2223
2224     Must(receivedWholeRequestBody); // or we should not be sending last-chunk
2225     flags.sentLastChunk = true;
2226
2227     typedef CommCbMemFunT<HttpStateData, CommIoCbParams> Dialer;
2228     requestSender = JobCallback(11,5, Dialer, this, HttpStateData::wroteLast);
2229     Comm::Write(serverConnection, "0\r\n\r\n", 5, requestSender, NULL);
2230     return true;
2231 }
2232
2233 void
2234 HttpStateData::doneSendingRequestBody()
2235 {
2236     ServerStateData::doneSendingRequestBody();
2237     debugs(11,5, HERE << serverConnection);
2238
2239     // do we need to write something after the last body byte?
2240     if (flags.chunked_request && finishingChunkedRequest())
2241         return;
2242     if (!flags.chunked_request && finishingBrokenPost())
2243         return;
2244
2245     sendComplete();
2246 }
2247
2248 // more origin request body data is available
2249 void
2250 HttpStateData::handleMoreRequestBodyAvailable()
2251 {
2252     if (eof || !Comm::IsConnOpen(serverConnection)) {
2253         // XXX: we should check this condition in other callbacks then!
2254         // TODO: Check whether this can actually happen: We should unsubscribe
2255         // as a body consumer when the above condition(s) are detected.
2256         debugs(11, 1, HERE << "Transaction aborted while reading HTTP body");
2257         return;
2258     }
2259
2260     assert(requestBodySource != NULL);
2261
2262     if (requestBodySource->buf().hasContent()) {
2263         // XXX: why does not this trigger a debug message on every request?
2264
2265         if (flags.headers_parsed && !flags.abuse_detected) {
2266             flags.abuse_detected = 1;
2267             debugs(11, 1, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request->client_addr << "' -> '" << entry->url() << "'" );
2268
2269             if (virginReply()->sline.status == HTTP_INVALID_HEADER) {
2270                 serverConnection->close();
2271                 return;
2272             }
2273         }
2274     }
2275
2276     HttpStateData::handleMoreRequestBodyAvailable();
2277 }
2278
2279 // premature end of the request body
2280 void
2281 HttpStateData::handleRequestBodyProducerAborted()
2282 {
2283     ServerStateData::handleRequestBodyProducerAborted();
2284     if (entry->isEmpty()) {
2285         debugs(11, 3, "request body aborted: " << serverConnection);
2286         // We usually get here when ICAP REQMOD aborts during body processing.
2287         // We might also get here if client-side aborts, but then our response
2288         // should not matter because either client-side will provide its own or
2289         // there will be no response at all (e.g., if the the client has left).
2290         ErrorState *err = new ErrorState(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR, fwd->request);
2291         err->detailError(ERR_DETAIL_SRV_REQMOD_REQ_BODY);
2292         fwd->fail(err);
2293     }
2294
2295     abortTransaction("request body producer aborted");
2296 }
2297
2298 // called when we wrote request headers(!) or a part of the body
2299 void
2300 HttpStateData::sentRequestBody(const CommIoCbParams &io)
2301 {
2302     if (io.size > 0)
2303         kb_incr(&statCounter.server.http.kbytes_out, io.size);
2304
2305     ServerStateData::sentRequestBody(io);
2306 }
2307
2308 // Quickly abort the transaction
2309 // TODO: destruction should be sufficient as the destructor should cleanup,
2310 // including canceling close handlers
2311 void
2312 HttpStateData::abortTransaction(const char *reason)
2313 {
2314     debugs(11,5, HERE << "aborting transaction for " << reason <<
2315            "; " << serverConnection << ", this " << this);
2316
2317     if (Comm::IsConnOpen(serverConnection)) {
2318         serverConnection->close();
2319         return;
2320     }
2321
2322     fwd->handleUnregisteredServerEnd();
2323     mustStop("HttpStateData::abortTransaction");
2324 }