src/http.cc

   1
   2 /*
   3  * $Id: http.cc,v 1.414 2003/05/24 11:32:34 robertc Exp $
   4  *
   5  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 /*
  37  * Anonymizing patch by lutz@as-node.jena.thur.de
  38  * have a look into http-anon.c to get more informations.
  39  */
  40
  41 #include "squid.h"
  42 #include "http.h"
  43 #include "authenticate.h"
  44 #include "Store.h"
  45 #include "HttpReply.h"
  46 #include "HttpRequest.h"
  47 #include "MemObject.h"
  48 #include "HttpHdrContRange.h"
  49 #include "ACLChecklist.h"
  50 #if DELAY_POOLS
  51 #include "DelayPools.h"
  52 #endif
  53
  54 CBDATA_TYPE(HttpStateData);
  55
  56
  57 static const char *const crlf = "\r\n";
  58
  59 static CWCB httpSendRequestEntity;
  60
  61 static IOCB httpReadReply;
  62 static void httpSendRequest(HttpStateData *);
  63 static PF httpStateFree;
  64 static PF httpTimeout;
  65 static void httpCacheNegatively(StoreEntry *);
  66 static void httpMakePrivate(StoreEntry *);
  67 static void httpMakePublic(StoreEntry *);
  68 static int httpCachableReply(HttpStateData *);
  69 static void httpMaybeRemovePublic(StoreEntry *, http_status);
  70 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, String strConnection, request_t * request, request_t * orig_request,
  71         HttpHeader * hdr_out, int we_do_ranges, http_state_flags);
  72 static int decideIfWeDoRanges (request_t * orig_request);
  73
  74
  75 static void
  76 httpStateFree(int fd, void *data)
  77 {
  78     HttpStateData *httpState = static_cast<HttpStateData *>(data);
  79
  80     if (httpState == NULL)
  81         return;
  82
  83     storeUnlockObject(httpState->entry);
  84
  85     if (httpState->reply_hdr) {
  86         memFree(httpState->reply_hdr, MEM_8K_BUF);
  87         httpState->reply_hdr = NULL;
  88     }
  89
  90     requestUnlink(httpState->request);
  91     requestUnlink(httpState->orig_request);
  92     httpState->request = NULL;
  93     httpState->orig_request = NULL;
  94     cbdataFree(httpState);
  95 }
  96
  97 int
  98 httpCachable(method_t method)
  99 {
 100     /* GET and HEAD are cachable. Others are not. */
 101
 102     if (method != METHOD_GET && method != METHOD_HEAD)
 103         return 0;
 104
 105     /* else cachable */
 106     return 1;
 107 }
 108
 109 static void
 110 httpTimeout(int fd, void *data)
 111 {
 112     HttpStateData *httpState = static_cast<HttpStateData *>(data);
 113     StoreEntry *entry = httpState->entry;
 114     debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry));
 115
 116     if (entry->store_status == STORE_PENDING) {
 117         if (entry->isEmpty()) {
 118             fwdFail(httpState->fwd,
 119                     errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT));
 120         }
 121     }
 122
 123     comm_close(fd);
 124 }
 125
 126 /* This object can be cached for a long time */
 127 static void
 128 httpMakePublic(StoreEntry * entry)
 129 {
 130     if (EBIT_TEST(entry->flags, ENTRY_CACHABLE))
 131         storeSetPublicKey(entry);
 132 }
 133
 134 /* This object should never be cached at all */
 135 static void
 136 httpMakePrivate(StoreEntry * entry)
 137 {
 138     storeExpireNow(entry);
 139     storeReleaseRequest(entry); /* delete object when not used */
 140     /* storeReleaseRequest clears ENTRY_CACHABLE flag */
 141 }
 142
 143 /* This object may be negatively cached */
 144 static void
 145 httpCacheNegatively(StoreEntry * entry)
 146 {
 147     storeNegativeCache(entry);
 148
 149     if (EBIT_TEST(entry->flags, ENTRY_CACHABLE))
 150         storeSetPublicKey(entry);
 151 }
 152
 153 static void
 154 httpMaybeRemovePublic(StoreEntry * e, http_status status)
 155 {
 156
 157     int remove
 158         = 0;
 159
 160     int forbidden = 0;
 161
 162     StoreEntry *pe;
 163
 164     if (!EBIT_TEST(e->flags, KEY_PRIVATE))
 165         return;
 166
 167     switch (status) {
 168
 169     case HTTP_OK:
 170
 171     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 172
 173     case HTTP_MULTIPLE_CHOICES:
 174
 175     case HTTP_MOVED_PERMANENTLY:
 176
 177     case HTTP_MOVED_TEMPORARILY:
 178
 179     case HTTP_GONE:
 180
 181     case HTTP_NOT_FOUND:
 182
 183         remove
 184             = 1;
 185
 186         break;
 187
 188     case HTTP_FORBIDDEN:
 189
 190     case HTTP_METHOD_NOT_ALLOWED:
 191         forbidden = 1;
 192
 193         break;
 194
 195 #if WORK_IN_PROGRESS
 196
 197     case HTTP_UNAUTHORIZED:
 198         forbidden = 1;
 199
 200         break;
 201
 202 #endif
 203
 204     default:
 205 #if QUESTIONABLE
 206         /*
 207          * Any 2xx response should eject previously cached entities...
 208          */
 209         if (status >= 200 && status < 300)
 210             remove
 211                 = 1;
 212
 213 #endif
 214
 215         break;
 216     }
 217
 218     if (!remove
 219             && !forbidden)
 220         return;
 221
 222     assert(e->mem_obj);
 223
 224     if (e->mem_obj->request)
 225         pe = storeGetPublicByRequest(e->mem_obj->request);
 226     else
 227         pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method);
 228
 229     if (pe != NULL) {
 230         assert(e != pe);
 231         storeRelease(pe);
 232     }
 233
 234     /*
 235      * Also remove any cached HEAD response in case the object has
 236      * changed.
 237      */
 238     if (e->mem_obj->request)
 239         pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_HEAD);
 240     else
 241         pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD);
 242
 243     if (pe != NULL) {
 244         assert(e != pe);
 245         storeRelease(pe);
 246     }
 247
 248     if (forbidden)
 249         return;
 250
 251     switch (e->mem_obj->method) {
 252
 253     case METHOD_PUT:
 254
 255     case METHOD_DELETE:
 256
 257     case METHOD_PROPPATCH:
 258
 259     case METHOD_MKCOL:
 260
 261     case METHOD_MOVE:
 262
 263     case METHOD_BMOVE:
 264
 265     case METHOD_BDELETE:
 266         /*
 267          * Remove any cached GET object if it is beleived that the
 268          * object may have changed as a result of other methods
 269          */
 270
 271         if (e->mem_obj->request)
 272             pe = storeGetPublicByRequestMethod(e->mem_obj->request, METHOD_GET);
 273         else
 274             pe = storeGetPublic(e->mem_obj->url, METHOD_GET);
 275
 276         if (pe != NULL) {
 277             assert(e != pe);
 278             storeRelease(pe);
 279         }
 280
 281         break;
 282
 283     default:
 284         /* Keep GCC happy. The methods above are all mutating HTTP methods
 285          */
 286         break;
 287     }
 288 }
 289
 290 void
 291 HttpStateData::processSurrogateControl(HttpReply *reply)
 292 {
 293 #if ESI
 294
 295     if (request->flags.accelerated && reply->surrogate_control) {
 296         HttpHdrScTarget *sctusable =
 297             httpHdrScGetMergedTarget(reply->surrogate_control,
 298                                      Config.Accel.surrogate_id);
 299
 300         if (sctusable) {
 301             if (EBIT_TEST(sctusable->mask, SC_NO_STORE) ||
 302                     (Config.onoff.surrogate_is_remote
 303                      && EBIT_TEST(sctusable->mask, SC_NO_STORE_REMOTE))) {
 304                 surrogateNoStore = true;
 305                 httpMakePrivate(entry);
 306             }
 307
 308             /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
 309              * accelerated request or not...
 310              * Still, this is an abtraction breach. - RC
 311              */
 312             if (sctusable->max_age != -1) {
 313                 if (sctusable->max_age < sctusable->max_stale)
 314                     reply->expires = reply->date + sctusable->max_age;
 315                 else
 316                     reply->expires = reply->date + sctusable->max_stale;
 317
 318                 /* And update the timestamps */
 319                 storeTimestampsSet(entry);
 320             }
 321
 322             /* We ignore cache-control directives as per the Surrogate specification */
 323             ignoreCacheControl = true;
 324
 325             httpHdrScTargetDestroy(sctusable);
 326         }
 327     }
 328
 329 #endif
 330 }
 331
 332 int
 333 cacheControlAllowsCaching(HttpHdrCc *cc)
 334 {
 335     if (cc) {
 336         const int cc_mask = cc->mask;
 337
 338         if (EBIT_TEST(cc_mask, CC_PRIVATE))
 339             return 0;
 340
 341         if (EBIT_TEST(cc_mask, CC_NO_CACHE))
 342             return 0;
 343
 344         if (EBIT_TEST(cc_mask, CC_NO_STORE))
 345             return 0;
 346     }
 347
 348     return 1;
 349 }
 350
 351 static int
 352 httpCachableReply(HttpStateData * httpState)
 353 {
 354     HttpReply const *rep = httpState->entry->getReply();
 355     HttpHeader const *hdr = &rep->header;
 356     const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0;
 357     const char *v;
 358
 359     if (httpState->surrogateNoStore)
 360         return 0;
 361
 362     if (!cacheControlAllowsCaching(rep->cache_control))
 363         return 0;
 364
 365     if (!httpState->ignoreCacheControl) {
 366         if (EBIT_TEST(cc_mask, CC_PRIVATE))
 367             return 0;
 368
 369         if (EBIT_TEST(cc_mask, CC_NO_CACHE))
 370             return 0;
 371
 372         if (EBIT_TEST(cc_mask, CC_NO_STORE))
 373             return 0;
 374     }
 375
 376     if (httpState->request->flags.auth) {
 377         /*
 378          * Responses to requests with authorization may be cached
 379          * only if a Cache-Control: public reply header is present.
 380          * RFC 2068, sec 14.9.4
 381          */
 382
 383         if (!EBIT_TEST(cc_mask, CC_PUBLIC))
 384             return 0;
 385     }
 386
 387     /* Pragma: no-cache in _replies_ is not documented in HTTP,
 388      * but servers like "Active Imaging Webcast/2.0" sure do use it */
 389     if (httpHeaderHas(hdr, HDR_PRAGMA)) {
 390         String s = httpHeaderGetList(hdr, HDR_PRAGMA);
 391         const int no_cache = strListIsMember(&s, "no-cache", ',');
 392         s.clean();
 393
 394         if (no_cache)
 395             return 0;
 396     }
 397
 398     /*
 399      * The "multipart/x-mixed-replace" content type is used for
 400      * continuous push replies.  These are generally dynamic and
 401      * probably should not be cachable
 402      */
 403     if ((v = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE)))
 404         if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
 405             return 0;
 406
 407     switch (httpState->entry->getReply()->sline.status) {
 408         /* Responses that are cacheable */
 409
 410     case HTTP_OK:
 411
 412     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 413
 414     case HTTP_MULTIPLE_CHOICES:
 415
 416     case HTTP_MOVED_PERMANENTLY:
 417
 418     case HTTP_GONE:
 419         /*
 420          * Don't cache objects that need to be refreshed on next request,
 421          * unless we know how to refresh it.
 422          */
 423
 424         if (!refreshIsCachable(httpState->entry))
 425             return 0;
 426
 427         /* don't cache objects from peers w/o LMT, Date, or Expires */
 428         /* check that is it enough to check headers @?@ */
 429         if (rep->date > -1)
 430             return 1;
 431         else if (rep->last_modified > -1)
 432             return 1;
 433         else if (!httpState->_peer)
 434             return 1;
 435
 436         /* @?@ (here and 302): invalid expires header compiles to squid_curtime */
 437         else if (rep->expires > -1)
 438             return 1;
 439         else
 440             return 0;
 441
 442         /* NOTREACHED */
 443         break;
 444
 445         /* Responses that only are cacheable if the server says so */
 446
 447     case HTTP_MOVED_TEMPORARILY:
 448         if (rep->expires > -1)
 449             return 1;
 450         else
 451             return 0;
 452
 453         /* NOTREACHED */
 454         break;
 455
 456         /* Errors can be negatively cached */
 457
 458     case HTTP_NO_CONTENT:
 459
 460     case HTTP_USE_PROXY:
 461
 462     case HTTP_BAD_REQUEST:
 463
 464     case HTTP_FORBIDDEN:
 465
 466     case HTTP_NOT_FOUND:
 467
 468     case HTTP_METHOD_NOT_ALLOWED:
 469
 470     case HTTP_REQUEST_URI_TOO_LARGE:
 471
 472     case HTTP_INTERNAL_SERVER_ERROR:
 473
 474     case HTTP_NOT_IMPLEMENTED:
 475
 476     case HTTP_BAD_GATEWAY:
 477
 478     case HTTP_SERVICE_UNAVAILABLE:
 479
 480     case HTTP_GATEWAY_TIMEOUT:
 481         return -1;
 482
 483         /* NOTREACHED */
 484         break;
 485
 486         /* Some responses can never be cached */
 487
 488     case HTTP_PARTIAL_CONTENT:  /* Not yet supported */
 489
 490     case HTTP_SEE_OTHER:
 491
 492     case HTTP_NOT_MODIFIED:
 493
 494     case HTTP_UNAUTHORIZED:
 495
 496     case HTTP_PROXY_AUTHENTICATION_REQUIRED:
 497
 498     case HTTP_INVALID_HEADER:   /* Squid header parsing error */
 499         return 0;
 500
 501     default:                    /* Unknown status code */
 502         debug (11,0)("httpCachableReply: unknown http status code in reply\n");
 503
 504         return 0;
 505
 506         /* NOTREACHED */
 507         break;
 508     }
 509
 510     /* NOTREACHED */
 511 }
 512
 513 /*
 514  * For Vary, store the relevant request headers as
 515  * virtual headers in the reply
 516  * Returns false if the variance cannot be stored
 517  */
 518 const char *
 519 httpMakeVaryMark(request_t * request, HttpReply const * reply)
 520 {
 521     String vary, hdr;
 522     const char *pos = NULL;
 523     const char *item;
 524     const char *value;
 525     int ilen;
 526     static String vstr;
 527
 528     vstr.clean();
 529     vary = httpHeaderGetList(&reply->header, HDR_VARY);
 530
 531     while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
 532         char *name = (char *)xmalloc(ilen + 1);
 533         xstrncpy(name, item, ilen + 1);
 534         Tolower(name);
 535         strListAdd(&vstr, name, ',');
 536         hdr = httpHeaderGetByName(&request->header, name);
 537         safe_free(name);
 538         value = hdr.buf();
 539
 540         if (value) {
 541             value = rfc1738_escape_part(value);
 542             vstr.append("=\"", 2);
 543             vstr.append(value);
 544             vstr.append("\"", 1);
 545         }
 546
 547         hdr.clean();
 548     }
 549
 550     vary.clean();
 551 #if X_ACCELERATOR_VARY
 552
 553     vary = httpHeaderGetList(&reply->header, HDR_X_ACCELERATOR_VARY);
 554
 555     while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
 556         char *name = (char *)xmalloc(ilen + 1);
 557         xstrncpy(name, item, ilen + 1);
 558         Tolower(name);
 559         strListAdd(&vstr, name, ',');
 560         hdr = httpHeaderGetByName(&request->header, name);
 561         safe_free(name);
 562         value = hdr.buf();
 563
 564         if (value) {
 565             value = rfc1738_escape_part(value);
 566             vstr.append("=\"", 2);
 567             vstr.append(value);
 568             vstr.append("\"", 1);
 569         }
 570
 571         hdr.clean();
 572     }
 573
 574     vary.clean();
 575 #endif
 576
 577     debug(11, 3) ("httpMakeVaryMark: %s\n", vstr.buf());
 578     return vstr.buf();
 579 }
 580
 581 /* rewrite this later using new interfaces @?@ */
 582 void
 583 HttpStateData::processReplyHeader(const char *buf, int size)
 584 {
 585     char *t = NULL;
 586     int room;
 587     size_t hdr_len;
 588     /* Creates a blank header. If this routine is made incremental, this will
 589      * not do
 590      */
 591     HttpReply *reply = httpReplyCreate();
 592     Ctx ctx;
 593     debug(11, 3) ("httpProcessReplyHeader: key '%s'\n",
 594                   entry->getMD5Text());
 595
 596     if (reply_hdr == NULL)
 597         reply_hdr = (char *)memAllocate(MEM_8K_BUF);
 598
 599     assert(reply_hdr_state == 0);
 600
 601     hdr_len = reply_hdr_size;
 602
 603     room = 8191 - hdr_len;
 604
 605     xmemcpy(reply_hdr + hdr_len, buf, room < size ? room : size);
 606
 607     hdr_len += room < size ? room : size;
 608
 609     reply_hdr[hdr_len] = '\0';
 610
 611     reply_hdr_size = hdr_len;
 612
 613     if (hdr_len > 4 && strncmp(reply_hdr, "HTTP/", 5)) {
 614         debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", reply_hdr);
 615         reply_hdr_state += 2;
 616         reply->sline.status = HTTP_INVALID_HEADER;
 617         storeEntryReplaceObject (entry, reply);
 618
 619         if (eof == 1) {
 620             fwdComplete(fwd);
 621             comm_close(fd);
 622         }
 623
 624         return;
 625     }
 626
 627     t = reply_hdr + hdr_len;
 628     /* headers can be incomplete only if object still arriving */
 629
 630     if (!eof) {
 631         size_t k = headersEnd(reply_hdr, 8192);
 632
 633         if (0 == k) {
 634             if (eof == 1) {
 635                 fwdComplete(fwd);
 636                 comm_close(fd);
 637             }
 638
 639             return;             /* headers not complete */
 640         }
 641
 642         t = reply_hdr + k;
 643     }
 644
 645     *t = '\0';
 646     reply_hdr_state++;
 647     assert(reply_hdr_state == 1);
 648     ctx = ctx_enter(entry->mem_obj->url);
 649     reply_hdr_state++;
 650     debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n",
 651                   reply_hdr);
 652     /* Parse headers into reply structure */
 653     /* what happens if we fail to parse here? */
 654     httpReplyParse(reply, reply_hdr, hdr_len);
 655     processSurrogateControl (reply);
 656     /* TODO: we need our own reply * in the httpState, as we probably don't want to replace
 657      * the storeEntry with interim headers
 658      */
 659
 660     /* TODO: IF the reply is a 1.0 reply, AND it has a Connection: Header
 661      * Parse the header and remove all referenced headers
 662      */
 663
 664     storeEntryReplaceObject(entry, reply);
 665     /* DO NOT USE reply now */
 666     reply = NULL;
 667
 668     if (entry->getReply()->content_range)
 669         currentOffset = entry->getReply()->content_range->spec.offset;
 670
 671     storeTimestampsSet(entry);
 672
 673     /* Check if object is cacheable or not based on reply code */
 674     debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", entry->getReply()->sline.status);
 675
 676     if (neighbors_do_private_keys)
 677         httpMaybeRemovePublic(entry, entry->getReply()->sline.status);
 678
 679     switch (httpCachableReply(this)) {
 680
 681     case 1:
 682
 683         if (httpHeaderHas(&entry->getReply()->header, HDR_VARY)
 684 #if X_ACCELERATOR_VARY
 685                 || httpHeaderHas(&entry->getReply()->header, HDR_X_ACCELERATOR_VARY)
 686 #endif
 687            ) {
 688             const char *vary = httpMakeVaryMark(orig_request, entry->getReply());
 689
 690             if (vary) {
 691                 entry->mem_obj->vary_headers = xstrdup(vary);
 692                 /* Kill the old base object if a change in variance is detected */
 693                 httpMakePublic(entry);
 694             } else {
 695                 httpMakePrivate(entry);
 696             }
 697         } else {
 698             httpMakePublic(entry);
 699         }
 700
 701         break;
 702
 703     case 0:
 704         httpMakePrivate(entry);
 705         break;
 706
 707     case -1:
 708         httpCacheNegatively(entry);
 709         break;
 710
 711     default:
 712         assert(0);
 713         break;
 714     }
 715
 716     if (!ignoreCacheControl && entry->getReply()->cache_control) {
 717         if (EBIT_TEST(entry->getReply()->cache_control->mask, CC_PROXY_REVALIDATE))
 718             EBIT_SET(entry->flags, ENTRY_REVALIDATE);
 719         else if (EBIT_TEST(entry->getReply()->cache_control->mask, CC_MUST_REVALIDATE))
 720             EBIT_SET(entry->flags, ENTRY_REVALIDATE);
 721     }
 722
 723     if (flags.keepalive)
 724         if (_peer)
 725             _peer->stats.n_keepalives_sent++;
 726
 727     if (entry->getReply()->keep_alive)
 728         if (_peer)
 729             _peer->stats.n_keepalives_recv++;
 730
 731     if (entry->getReply()->date > -1 && !_peer) {
 732         int skew = abs(entry->getReply()->date - squid_curtime);
 733
 734         if (skew > 86400)
 735             debug(11, 3) ("%s's clock is skewed by %d seconds!\n",
 736                           request->host, skew);
 737     }
 738
 739     ctx_exit(ctx);
 740 #if HEADERS_LOG
 741
 742     headersLog(1, 0, request->method, entry->getReply());
 743 #endif
 744
 745     if (eof == 1) {
 746         fwdComplete(fwd);
 747         comm_close(fd);
 748     }
 749 }
 750
 751 HttpStateData::ConnectionStatus
 752 HttpStateData::statusIfComplete() const
 753 {
 754     HttpReply const *reply = entry->getReply();
 755     /* If the reply wants to close the connection, it takes precedence */
 756
 757     if (httpHeaderHasConnDir(&reply->header, "close"))
 758         return COMPLETE_NONPERSISTENT_MSG;
 759
 760     /* If we didn't send a keep-alive request header, then this
 761      * can not be a persistent connection.
 762      */
 763     if (!flags.keepalive)
 764         return COMPLETE_NONPERSISTENT_MSG;
 765
 766     /*
 767      * What does the reply have to say about keep-alive?
 768      */
 769     /*
 770      * XXX BUG?
 771      * If the origin server (HTTP/1.0) does not send a keep-alive
 772      * header, but keeps the connection open anyway, what happens?
 773      * We'll return here and http.c waits for an EOF before changing
 774      * store_status to STORE_OK.   Combine this with ENTRY_FWD_HDR_WAIT
 775      * and an error status code, and we might have to wait until
 776      * the server times out the socket.
 777      */
 778     if (!reply->keep_alive)
 779         return COMPLETE_NONPERSISTENT_MSG;
 780
 781     return COMPLETE_PERSISTENT_MSG;
 782 }
 783
 784 HttpStateData::ConnectionStatus
 785 HttpStateData::persistentConnStatus() const
 786 {
 787     HttpReply const *reply = entry->getReply();
 788     int clen;
 789     debug(11, 3) ("httpPconnTransferDone: FD %d\n", fd);
 790     ConnectionStatus result = statusIfComplete();
 791     debug(11, 5) ("httpPconnTransferDone: content_length=%d\n",
 792                   reply->content_length);
 793     /* If we haven't seen the end of reply headers, we are not done */
 794
 795     if (reply_hdr_state < 2)
 796         return INCOMPLETE_MSG;
 797
 798     clen = httpReplyBodySize(request->method, reply);
 799
 800     /* If there is no message body, we can be persistent */
 801     if (0 == clen)
 802         return result;
 803
 804     /* If the body size is unknown we must wait for EOF */
 805     if (clen < 0)
 806         return INCOMPLETE_MSG;
 807
 808     /* If the body size is known, we must wait until we've gotten all of it.  */
 809     if (entry->mem_obj->endOffset() < reply->content_length + reply->hdr_sz)
 810         return INCOMPLETE_MSG;
 811
 812     /* We got it all */
 813     return result;
 814 }
 815
 816 /* This will be called when data is ready to be read from fd.  Read until
 817  * error or connection closed. */
 818 /* XXX this function is too long! */
 819 static void
 820 httpReadReply(int fd, char *buf, size_t len, comm_err_t flag, int xerrno,void *data)
 821 {
 822     HttpStateData *httpState = static_cast<HttpStateData *>(data);
 823     assert (fd == httpState->fd);
 824     httpState->readReply (fd, buf, len, flag, xerrno, data);
 825 }
 826
 827 void
 828 HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, int xerrno,void *data)
 829 {
 830     int bin;
 831     int clen;
 832     do_next_read = 0;
 833
 834
 835     assert(buf == readBuf);
 836
 837     /* Bail out early on COMM_ERR_CLOSING - close handlers will tidy up for us
 838     */
 839
 840     if (flag == COMM_ERR_CLOSING) {
 841         debug (11,1)("http socket closing\n");
 842         return;
 843     }
 844
 845     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 846         maybeReadData();
 847         return;
 848     }
 849
 850     errno = 0;
 851     /* prepare the read size for the next read (if any) */
 852 #if DELAY_POOLS
 853
 854     DelayId delayId = entry->mem_obj->mostBytesAllowed();
 855
 856 #endif
 857
 858     debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, (int)len);
 859
 860     if (flag == COMM_OK && len > 0) {
 861 #if DELAY_POOLS
 862         delayId.bytesIn(len);
 863 #endif
 864
 865         kb_incr(&statCounter.server.all.kbytes_in, len);
 866         kb_incr(&statCounter.server.http.kbytes_in, len);
 867         commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
 868         IOStats.Http.reads++;
 869
 870         for (clen = len - 1, bin = 0; clen; bin++)
 871             clen >>= 1;
 872
 873         IOStats.Http.read_hist[bin]++;
 874     }
 875
 876     if (!reply_hdr && flag == COMM_OK && len > 0) {
 877         /* Skip whitespace */
 878
 879         while (len > 0 && xisspace(*buf))
 880             xmemmove(buf, buf + 1, len--);
 881
 882         if (len == 0) {
 883             /* Continue to read... */
 884             do_next_read = 1;
 885             maybeReadData();
 886             return;
 887         }
 888     }
 889
 890     if (flag != COMM_OK || len < 0) {
 891         debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n",
 892                       fd, xstrerror());
 893
 894         if (ignoreErrno(errno)) {
 895             do_next_read = 1;
 896         } else if (entry->isEmpty()) {
 897             ErrorState *err;
 898             err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR);
 899             err->request = requestLink((request_t *) request);
 900             err->xerrno = errno;
 901             fwdFail(fwd, err);
 902             do_next_read = 0;
 903             comm_close(fd);
 904         } else {
 905             do_next_read = 0;
 906             comm_close(fd);
 907         }
 908     } else if (flag == COMM_OK && len == 0 && entry->isEmpty()) {
 909         ErrorState *err;
 910         err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE);
 911         err->xerrno = errno;
 912         err->request = requestLink((request_t *) request);
 913         fwdFail(fwd, err);
 914         eof = 1;
 915         do_next_read = 0;
 916         comm_close(fd);
 917     } else if (flag == COMM_OK && len == 0) {
 918         /* Connection closed; retrieval done. */
 919         eof = 1;
 920
 921         if (reply_hdr_state < 2)
 922             /*
 923              * Yes Henrik, there is a point to doing this.  When we
 924              * called httpProcessReplyHeader() before, we didn't find
 925              * the end of headers, but now we are definately at EOF, so
 926              * we want to process the reply headers.
 927              */
 928             /* doesn't return */
 929             processReplyHeader(buf, len);
 930         else {
 931             fwdComplete(fwd);
 932             do_next_read = 0;
 933             comm_close(fd);
 934         }
 935     } else {
 936         if (reply_hdr_state < 2) {
 937             processReplyHeader(buf, len);
 938
 939             if (reply_hdr_state == 2) {
 940                 http_status s = entry->getReply()->sline.status;
 941 #if WIP_FWD_LOG
 942
 943                 fwdStatus(fwd, s);
 944 #endif
 945                 /*
 946                  * If its not a reply that we will re-forward, then
 947                  * allow the client to get it.
 948                  */
 949
 950                 if (!fwdReforwardableStatus(s))
 951                     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 952             }
 953         }
 954
 955         processReplyData(buf, len);
 956     }
 957 }
 958
 959 void
 960 HttpStateData::processReplyData(const char *buf, size_t len)
 961 {
 962     if (reply_hdr_state < 2) {
 963         do_next_read = 1;
 964         maybeReadData();
 965         return;
 966     }
 967
 968     StoreIOBuffer tempBuffer;
 969
 970     if (!flags.headers_pushed) {
 971         /* The first block needs us to skip the headers */
 972         /* TODO: make this cleaner. WE should push the headers, NOT the parser */
 973         size_t end = headersEnd (buf, len);
 974         /* IF len > end, we need to append data after the
 975          * out of band update to the store
 976          */
 977
 978         if (len > end) {
 979             tempBuffer.data = (char *)buf+end;
 980             tempBuffer.length = len - end;
 981             tempBuffer.offset = currentOffset;
 982             currentOffset += tempBuffer.length;
 983             entry->write (tempBuffer);
 984         }
 985
 986         flags.headers_pushed = 1;
 987     } else {
 988         tempBuffer.data = (char *)buf;
 989         tempBuffer.length = len;
 990         tempBuffer.offset = currentOffset;
 991         currentOffset += len;
 992         entry->write(tempBuffer);
 993     }
 994
 995     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 996         /*
 997          * the above storeAppend() call could ABORT this entry,
 998          * in that case, the server FD should already be closed.
 999          * there's nothing for us to do.
1000          */
1001         (void) 0;
1002     } else
1003         switch (persistentConnStatus()) {
1004
1005         case INCOMPLETE_MSG:
1006             /* Wait for EOF condition */
1007             do_next_read = 1;
1008             break;
1009
1010         case COMPLETE_PERSISTENT_MSG:
1011             /* yes we have to clear all these! */
1012             commSetTimeout(fd, -1, NULL, NULL);
1013             do_next_read = 0;
1014
1015             comm_remove_close_handler(fd, httpStateFree, this);
1016             fwdUnregister(fd, fwd);
1017             pconnPush(fd, request->host, request->port);
1018             fwdComplete(fwd);
1019             fd = -1;
1020             httpStateFree(fd, this);
1021             return;
1022
1023         case COMPLETE_NONPERSISTENT_MSG:
1024             /* close the connection ourselves */
1025             /* yes - same as for a complete persistent conn here */
1026             commSetTimeout(fd, -1, NULL, NULL);
1027             commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0);
1028             comm_remove_close_handler(fd, httpStateFree, this);
1029             fwdUnregister(fd, fwd);
1030             fwdComplete(fwd);
1031             /* TODO: check that fd is still open here */
1032             comm_close (fd);
1033             fd = -1;
1034             httpStateFree(fd, this);
1035             return;
1036         }
1037
1038     maybeReadData();
1039 }
1040
1041 void
1042 HttpStateData::maybeReadData()
1043 {
1044     if (do_next_read) {
1045         do_next_read = 0;
1046         entry->delayAwareRead(fd, buf, SQUID_TCP_SO_RCVBUF, httpReadReply, this);
1047     }
1048 }
1049
1050 /* This will be called when request write is complete. Schedule read of
1051  * reply. */
1052 void
1053 HttpStateData::SendComplete(int fd, char *bufnotused, size_t size, comm_err_t errflag, void *data)
1054 {
1055     HttpStateData *httpState = static_cast<HttpStateData *>(data);
1056     StoreEntry *entry = httpState->entry;
1057     ErrorState *err;
1058     debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n",
1059                   fd, (int) size, errflag);
1060 #if URL_CHECKSUM_DEBUG
1061
1062     entry->mem_obj->checkUrlChecksum();
1063 #endif
1064
1065     if (size > 0) {
1066         fd_bytes(fd, size, FD_WRITE);
1067         kb_incr(&statCounter.server.all.kbytes_out, size);
1068         kb_incr(&statCounter.server.http.kbytes_out, size);
1069     }
1070
1071     if (errflag == COMM_ERR_CLOSING)
1072         return;
1073
1074     if (errflag) {
1075         err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR);
1076         err->xerrno = errno;
1077         err->request = requestLink(httpState->orig_request);
1078         errorAppendEntry(entry, err);
1079         comm_close(fd);
1080         return;
1081     } else {
1082         /* Schedule read reply. */
1083         entry->delayAwareRead(fd, httpState->buf, SQUID_TCP_SO_RCVBUF, httpReadReply, httpState);
1084         /*
1085          * Set the read timeout here because it hasn't been set yet.
1086          * We only set the read timeout after the request has been
1087          * fully written to the server-side.  If we start the timeout
1088          * after connection establishment, then we are likely to hit
1089          * the timeout for POST/PUT requests that have very large
1090          * request bodies.
1091          */
1092         commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState);
1093     }
1094 }
1095
1096 /*
1097  * build request headers and append them to a given MemBuf
1098  * used by httpBuildRequestPrefix()
1099  * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing
1100  */
1101 void
1102 httpBuildRequestHeader(request_t * request,
1103                        request_t * orig_request,
1104                        StoreEntry * entry,
1105                        HttpHeader * hdr_out,
1106                        http_state_flags flags)
1107 {
1108     /* building buffer for complex strings */
1109 #define BBUF_SZ (MAX_URL+32)
1110     LOCAL_ARRAY(char, bbuf, BBUF_SZ);
1111     const HttpHeader *hdr_in = &orig_request->header;
1112     const HttpHeaderEntry *e;
1113     String strFwd;
1114     HttpHeaderPos pos = HttpHeaderInitPos;
1115     httpHeaderInit(hdr_out, hoRequest);
1116     /* append our IMS header */
1117
1118     if (request->lastmod > -1 && request->method == METHOD_GET)
1119         httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod);
1120
1121     bool we_do_ranges = decideIfWeDoRanges (orig_request);
1122
1123     String strConnection (httpHeaderGetList(hdr_in, HDR_CONNECTION));
1124
1125     while ((e = httpHeaderGetEntry(hdr_in, &pos)))
1126         copyOneHeaderFromClientsideRequestToUpstreamRequest(e, strConnection, request, orig_request, hdr_out, we_do_ranges, flags);
1127
1128     /* Abstraction break: We should interpret multipart/byterange responses
1129      * into offset-length data, and this works around our inability to do so.
1130      */
1131     if (!we_do_ranges && orig_request->multipartRangeRequest()) {
1132         /* don't cache the result */
1133         orig_request->flags.cachable = 0;
1134         /* pretend it's not a range request */
1135         orig_request->range->deleteSelf();
1136         orig_request->range = NULL;
1137         orig_request->flags.range = 0;
1138     }
1139
1140
1141     /* append Via */
1142     if (Config.onoff.via) {
1143         String strVia;
1144         strVia = httpHeaderGetList(hdr_in, HDR_VIA);
1145         snprintf(bbuf, BBUF_SZ, "%d.%d %s",
1146                  orig_request->http_ver.major,
1147                  orig_request->http_ver.minor, ThisCache);
1148         strListAdd(&strVia, bbuf, ',');
1149         httpHeaderPutStr(hdr_out, HDR_VIA, strVia.buf());
1150         strVia.clean();
1151     }
1152
1153 #if ESI
1154     {
1155         /* Append Surrogate-Capabilities */
1156         String strSurrogate (httpHeaderGetList(hdr_in, HDR_SURROGATE_CAPABILITY));
1157         snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0 ESI/1.0\"",
1158                  Config.Accel.surrogate_id);
1159         strListAdd(&strSurrogate, bbuf, ',');
1160         httpHeaderPutStr(hdr_out, HDR_SURROGATE_CAPABILITY, strSurrogate.buf());
1161     }
1162 #endif
1163
1164     /* append X-Forwarded-For */
1165     strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR);
1166
1167     if (opt_forwarded_for && orig_request->client_addr.s_addr != no_addr.s_addr)
1168         strListAdd(&strFwd, inet_ntoa(orig_request->client_addr), ',');
1169     else
1170         strListAdd(&strFwd, "unknown", ',');
1171
1172     httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strFwd.buf());
1173
1174     strFwd.clean();
1175
1176     /* append Host if not there already */
1177     if (!httpHeaderHas(hdr_out, HDR_HOST)) {
1178         if (orig_request->peer_domain) {
1179             httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->peer_domain);
1180         } else if (orig_request->port == urlDefaultPort(orig_request->protocol)) {
1181             /* use port# only if not default */
1182             httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host);
1183         } else {
1184             httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d",
1185                               orig_request->host, (int) orig_request->port);
1186         }
1187     }
1188
1189     /* append Authorization if known in URL, not in header and going direct */
1190     if (!httpHeaderHas(hdr_out, HDR_AUTHORIZATION)) {
1191         if (!request->flags.proxying && *request->login) {
1192             httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s",
1193                               base64_encode(request->login));
1194         }
1195     }
1196
1197     /* append Proxy-Authorization if configured for peer, and proxying */
1198     if (request->flags.proxying && orig_request->peer_login &&
1199             !httpHeaderHas(hdr_out, HDR_PROXY_AUTHORIZATION)) {
1200         if (*orig_request->peer_login == '*') {
1201             /* Special mode, to pass the username to the upstream cache */
1202             char loginbuf[256];
1203             const char *username = "-";
1204
1205             if (orig_request->auth_user_request)
1206                 username = orig_request->auth_user_request->username();
1207
1208             snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1);
1209
1210             httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s",
1211                               base64_encode(loginbuf));
1212         } else if (strcmp(orig_request->peer_login, "PASS") == 0) {
1213             /* Nothing to do */
1214         } else if (strcmp(orig_request->peer_login, "PROXYPASS") == 0) {
1215             /* Nothing to do */
1216         } else {
1217             httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s",
1218                               base64_encode(orig_request->peer_login));
1219         }
1220     }
1221
1222     /* append WWW-Authorization if configured for peer */
1223     if (flags.originpeer && orig_request->peer_login &&
1224             !httpHeaderHas(hdr_out, HDR_AUTHORIZATION)) {
1225         if (strcmp(orig_request->peer_login, "PASS") == 0) {
1226             /* No credentials to forward.. (should have been done above if available) */
1227         } else if (strcmp(orig_request->peer_login, "PROXYPASS") == 0) {
1228             /* Special mode, convert proxy authentication to WWW authentication
1229              */
1230             const char *auth = httpHeaderGetStr(hdr_in, HDR_PROXY_AUTHORIZATION);
1231
1232             if (auth && strncasecmp(auth, "basic ", 6) == 0) {
1233                 httpHeaderPutStr(hdr_out, HDR_AUTHORIZATION, auth);
1234             }
1235         } else if (*orig_request->peer_login == '*') {
1236             /* Special mode, to pass the username to the upstream cache */
1237             char loginbuf[256];
1238             const char *username = "-";
1239
1240             if (orig_request->auth_user_request)
1241                 username = authenticateUserRequestUsername(orig_request->auth_user_request);
1242
1243             snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1);
1244
1245             httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s",
1246                               base64_encode(loginbuf));
1247         } else {
1248             /* Fixed login string */
1249             httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s",
1250                               base64_encode(orig_request->peer_login));
1251         }
1252     }
1253
1254     /* append Cache-Control, add max-age if not there already */
1255     {
1256         HttpHdrCc *cc = httpHeaderGetCc(hdr_in);
1257
1258         if (!cc)
1259             cc = httpHdrCcCreate();
1260
1261         if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) {
1262             const char *url =
1263                 entry ? storeUrl(entry) : urlCanonical(orig_request);
1264             httpHdrCcSetMaxAge(cc, getMaxAge(url));
1265
1266             if (request->urlpath.size())
1267                 assert(strstr(url, request->urlpath.buf()));
1268         }
1269
1270         if (flags.only_if_cached)
1271             EBIT_SET(cc->mask, CC_ONLY_IF_CACHED);
1272
1273         httpHeaderPutCc(hdr_out, cc);
1274
1275         httpHdrCcDestroy(cc);
1276     }
1277
1278     /* maybe append Connection: keep-alive */
1279     if (flags.keepalive) {
1280         if (flags.proxying) {
1281             httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive");
1282         } else {
1283             httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive");
1284         }
1285     }
1286
1287     /* append Front-End-Https */
1288     if (flags.front_end_https) {
1289         if (flags.front_end_https == 1 || request->protocol == PROTO_HTTPS)
1290             httpHeaderPutStr(hdr_out, HDR_FRONT_END_HTTPS, "On");
1291     }
1292
1293     /* Now mangle the headers. */
1294     httpHdrMangleList(hdr_out, request);
1295
1296     strConnection.clean();
1297 }
1298
1299
1300 void
1301 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, String strConnection, request_t * request, request_t * orig_request, HttpHeader * hdr_out, int we_do_ranges, http_state_flags flags)
1302 {
1303     debug(11, 5) ("httpBuildRequestHeader: %s: %s\n",
1304                   e->name.buf(), e->value.buf());
1305
1306     if (!httpRequestHdrAllowed(e, &strConnection)) {
1307         debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n",+       e->name.buf());
1308         return;
1309     }
1310
1311     switch (e->id) {
1312
1313     case HDR_PROXY_AUTHORIZATION:
1314         /* Only pass on proxy authentication to peers for which
1315          * authentication forwarding is explicitly enabled
1316          */
1317
1318         if (flags.proxying && orig_request->peer_login &&
1319                 strcmp(orig_request->peer_login, "PASS") == 0) {
1320             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1321         }
1322
1323         break;
1324
1325     case HDR_AUTHORIZATION:
1326         /* Pass on WWW authentication */
1327
1328         if (!flags.originpeer) {
1329             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1330         } else {
1331             /* In accelerators, only forward authentication if enabled
1332              * (see also below for proxy->server authentication)
1333              */
1334
1335             if (orig_request->peer_login && (strcmp(orig_request->peer_login, "PASS") == 0 || strcmp(orig_request->peer_login, "PROXYPASS") == 0)) {
1336                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1337             }
1338         }
1339
1340         break;
1341
1342     case HDR_HOST:
1343         /*
1344          * Normally Squid does not copy the Host: header from
1345          * a client request into the forwarded request headers.
1346          * However, there is one case when we do: If the URL
1347          * went through our redirector and the admin configured
1348          * 'redir_rewrites_host' to be off.
1349          */
1350
1351         if (request->flags.redirected)
1352             if (!Config.onoff.redir_rewrites_host)
1353                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1354
1355         break;
1356
1357     case HDR_IF_MODIFIED_SINCE:
1358         /* append unless we added our own;
1359          * note: at most one client's ims header can pass through */
1360         if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE))
1361             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1362
1363         break;
1364
1365     case HDR_MAX_FORWARDS:
1366         if (orig_request->method == METHOD_TRACE) {
1367             const int hops = httpHeaderEntryGetInt(e);
1368
1369             if (hops > 0)
1370                 httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1);
1371         }
1372
1373         break;
1374
1375     case HDR_VIA:
1376         /* If Via is disabled then forward any received header as-is */
1377
1378         if (!Config.onoff.via)
1379             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1380
1381         break;
1382
1383     case HDR_RANGE:
1384
1385     case HDR_IF_RANGE:
1386
1387     case HDR_REQUEST_RANGE:
1388         if (!we_do_ranges)
1389             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1390
1391         break;
1392
1393     case HDR_PROXY_CONNECTION:
1394
1395     case HDR_CONNECTION:
1396
1397     case HDR_X_FORWARDED_FOR:
1398
1399     case HDR_CACHE_CONTROL:
1400         /* append these after the loop if needed */
1401         break;
1402
1403     case HDR_FRONT_END_HTTPS:
1404         if (!flags.front_end_https)
1405             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1406
1407         break;
1408
1409     default:
1410         /* pass on all other header fields */
1411         httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
1412     }
1413 }
1414
1415 int
1416 decideIfWeDoRanges (request_t * orig_request)
1417 {
1418     int result = 1;
1419     /* decide if we want to do Ranges ourselves
1420      * and fetch the whole object now)
1421      * We want to handle Ranges ourselves iff
1422      *    - we can actually parse client Range specs
1423      *    - the specs are expected to be simple enough (e.g. no out-of-order ranges)
1424      *    - reply will be cachable
1425      * (If the reply will be uncachable we have to throw it away after
1426      *  serving this request, so it is better to forward ranges to
1427      *  the server and fetch only the requested content)
1428      */
1429
1430     if (NULL == orig_request->range || !orig_request->flags.cachable
1431             || orig_request->range->offsetLimitExceeded())
1432         result = 0;
1433
1434     debug(11, 8) ("decideIfWeDoRanges: range specs: %p, cachable: %d; we_do_ranges: %d\n",
1435                   orig_request->range, orig_request->flags.cachable, result);
1436
1437     return result;
1438 }
1439
1440
1441 /* build request prefix and append it to a given MemBuf;
1442  * return the length of the prefix */
1443 mb_size_t
1444 httpBuildRequestPrefix(request_t * request,
1445                        request_t * orig_request,
1446                        StoreEntry * entry,
1447                        MemBuf * mb,
1448                        http_state_flags flags)
1449 {
1450     const int offset = mb->size;
1451     http_version_t httpver;
1452     httpBuildVersion(&httpver, 1, 0);
1453     memBufPrintf(mb, "%s %s HTTP/%d.%d\r\n",
1454                  RequestMethodStr[request->method],
1455                  request->urlpath.size() ? request->urlpath.buf() : "/",
1456                  httpver.major,httpver.minor);
1457     /* build and pack headers */
1458     {
1459         HttpHeader hdr;
1460         Packer p;
1461         httpBuildRequestHeader(request, orig_request, entry, &hdr, flags);
1462         packerToMemInit(&p, mb);
1463         httpHeaderPackInto(&hdr, &p);
1464         httpHeaderClean(&hdr);
1465         packerClean(&p);
1466     }
1467     /* append header terminator */
1468     memBufAppend(mb, crlf, 2);
1469     return mb->size - offset;
1470 }
1471
1472 /* This will be called when connect completes. Write request. */
1473 static void
1474 httpSendRequest(HttpStateData * httpState)
1475 {
1476     MemBuf mb;
1477     request_t *req = httpState->request;
1478     StoreEntry *entry = httpState->entry;
1479     peer *p = httpState->_peer;
1480     CWCB *sendHeaderDone;
1481
1482     debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", httpState->fd,
1483                   httpState);
1484
1485     if (httpState->orig_request->body_connection)
1486         sendHeaderDone = httpSendRequestEntity;
1487     else
1488         sendHeaderDone = HttpStateData::SendComplete;
1489
1490     if (p != NULL) {
1491         if (p->options.originserver) {
1492             httpState->flags.proxying = 0;
1493             httpState->flags.originpeer = 1;
1494         } else {
1495             httpState->flags.proxying = 1;
1496             httpState->flags.originpeer = 0;
1497         }
1498     } else {
1499         httpState->flags.proxying = 0;
1500         httpState->flags.originpeer = 0;
1501     }
1502
1503     /*
1504      * Is keep-alive okay for all request methods?
1505      */
1506     if (!Config.onoff.server_pconns)
1507         httpState->flags.keepalive = 0;
1508     else if (p == NULL)
1509         httpState->flags.keepalive = 1;
1510     else if (p->stats.n_keepalives_sent < 10)
1511         httpState->flags.keepalive = 1;
1512     else if ((double) p->stats.n_keepalives_recv /
1513              (double) p->stats.n_keepalives_sent > 0.50)
1514         httpState->flags.keepalive = 1;
1515
1516     if (httpState->_peer) {
1517         if (neighborType(httpState->_peer, httpState->request) == PEER_SIBLING &&
1518                 !httpState->_peer->options.allow_miss)
1519             httpState->flags.only_if_cached = 1;
1520
1521         httpState->flags.front_end_https = httpState->_peer->front_end_https;
1522     }
1523
1524     memBufDefInit(&mb);
1525     httpBuildRequestPrefix(req,
1526                            httpState->orig_request,
1527                            entry,
1528                            &mb,
1529                            httpState->flags);
1530     debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", httpState->fd, mb.buf);
1531     comm_old_write_mbuf(httpState->fd, mb, sendHeaderDone, httpState);
1532 }
1533
1534 void
1535 httpStart(FwdState * fwd)
1536 {
1537     int fd = fwd->server_fd;
1538     HttpStateData *httpState;
1539     request_t *proxy_req;
1540     request_t *orig_req = fwd->request;
1541     debug(11, 3) ("httpStart: \"%s %s\"\n",
1542                   RequestMethodStr[orig_req->method],
1543                   storeUrl(fwd->entry));
1544     CBDATA_INIT_TYPE(HttpStateData);
1545     httpState = cbdataAlloc(HttpStateData);
1546     httpState->ignoreCacheControl = false;
1547     httpState->surrogateNoStore = false;
1548     storeLockObject(fwd->entry);
1549     httpState->fwd = fwd;
1550     httpState->entry = fwd->entry;
1551     httpState->fd = fd;
1552
1553     if (fwd->servers)
1554         httpState->_peer = fwd->servers->_peer;         /* might be NULL */
1555
1556     if (httpState->_peer) {
1557         const char *url;
1558
1559         if (httpState->_peer->options.originserver)
1560             url = orig_req->urlpath.buf();
1561         else
1562             url = storeUrl(httpState->entry);
1563
1564         proxy_req = requestCreate(orig_req->method,
1565                                   orig_req->protocol, url);
1566
1567         xstrncpy(proxy_req->host, httpState->_peer->host, SQUIDHOSTNAMELEN);
1568
1569         proxy_req->port = httpState->_peer->http_port;
1570
1571         proxy_req->flags = orig_req->flags;
1572
1573         proxy_req->lastmod = orig_req->lastmod;
1574
1575         httpState->request = requestLink(proxy_req);
1576
1577         httpState->orig_request = requestLink(orig_req);
1578
1579         proxy_req->flags.proxying = 1;
1580
1581         /*
1582          * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
1583          * We might end up getting the object from somewhere else if,
1584          * for example, the request to this neighbor fails.
1585          */
1586         if (httpState->_peer->options.proxy_only)
1587             storeReleaseRequest(httpState->entry);
1588
1589 #if DELAY_POOLS
1590
1591         httpState->entry->setNoDelay(httpState->_peer->options.no_delay);
1592
1593 #endif
1594
1595     } else {
1596         httpState->request = requestLink(orig_req);
1597         httpState->orig_request = requestLink(orig_req);
1598     }
1599
1600     /*
1601      * register the handler to free HTTP state data when the FD closes
1602      */
1603     comm_add_close_handler(fd, httpStateFree, httpState);
1604
1605     statCounter.server.all.requests++;
1606
1607     statCounter.server.http.requests++;
1608
1609     httpSendRequest(httpState);
1610
1611     /*
1612      * We used to set the read timeout here, but not any more.
1613      * Now its set in httpSendComplete() after the full request,
1614      * including request body, has been written to the server.
1615      */
1616 }
1617
1618 static void
1619 httpSendRequestEntityDone(int fd, void *data)
1620 {
1621     HttpStateData *httpState = static_cast<HttpStateData *>(data);
1622     ACLChecklist ch;
1623     debug(11, 5) ("httpSendRequestEntityDone: FD %d\n", fd);
1624     ch.request = requestLink(httpState->request);
1625
1626     if (!Config.accessList.brokenPosts) {
1627         debug(11, 5) ("httpSendRequestEntityDone: No brokenPosts list\n");
1628         HttpStateData::SendComplete(fd, NULL, 0, COMM_OK, data);
1629     } else if (!aclCheckFast(Config.accessList.brokenPosts, &ch)) {
1630         debug(11, 5) ("httpSendRequestEntityDone: didn't match brokenPosts\n");
1631         HttpStateData::SendComplete(fd, NULL, 0, COMM_OK, data);
1632     } else {
1633         debug(11, 2) ("httpSendRequestEntityDone: matched brokenPosts\n");
1634         comm_old_write(fd, "\r\n", 2, HttpStateData::SendComplete, data, NULL);
1635     }
1636 }
1637
1638 static void
1639 httpRequestBodyHandler(char *buf, ssize_t size, void *data)
1640 {
1641     HttpStateData *httpState = (HttpStateData *) data;
1642
1643     if (size > 0) {
1644         comm_old_write(httpState->fd, buf, size, httpSendRequestEntity, data, memFree8K);
1645     } else if (size == 0) {
1646         /* End of body */
1647         memFree8K(buf);
1648         httpSendRequestEntityDone(httpState->fd, data);
1649     } else {
1650         /* Failed to get whole body, probably aborted */
1651         memFree8K(buf);
1652         HttpStateData::SendComplete(httpState->fd, NULL, 0, COMM_ERR_CLOSING, data);
1653     }
1654 }
1655
1656 static void
1657 httpSendRequestEntity(int fd, char *bufnotused, size_t size, comm_err_t errflag, void *data)
1658 {
1659     HttpStateData *httpState = static_cast<HttpStateData *>(data);
1660     StoreEntry *entry = httpState->entry;
1661     ErrorState *err;
1662     debug(11, 5) ("httpSendRequestEntity: FD %d: size %d: errflag %d.\n",
1663                   fd, (int) size, errflag);
1664
1665     if (size > 0) {
1666         fd_bytes(fd, size, FD_WRITE);
1667         kb_incr(&statCounter.server.all.kbytes_out, size);
1668         kb_incr(&statCounter.server.http.kbytes_out, size);
1669     }
1670
1671     if (errflag == COMM_ERR_CLOSING)
1672         return;
1673
1674     if (errflag) {
1675         err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR);
1676         err->xerrno = errno;
1677         err->request = requestLink(httpState->orig_request);
1678         errorAppendEntry(entry, err);
1679         comm_close(fd);
1680         return;
1681     }
1682
1683     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1684         comm_close(fd);
1685         return;
1686     }
1687
1688     clientReadBody(httpState->orig_request, (char *)memAllocate(MEM_8K_BUF), 8192, httpRequestBodyHandler, httpState);
1689 }
1690
1691 void
1692 httpBuildVersion(http_version_t * version, unsigned int major, unsigned int minor)
1693 {
1694     version->major = major;
1695     version->minor = minor;
1696 }