src/http.cc

   1
   2 /*
   3  * $Id: http.cc,v 1.321 1998/09/15 22:05:09 wessels Exp $
   4  *
   5  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  Duane Wessels and the University of California San Diego.  Please
  16  *  see the COPYRIGHT file for full details.  Squid incorporates
  17  *  software developed and/or copyrighted by other sources.  Please see
  18  *  the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 /*
  37  * Anonymizing patch by lutz@as-node.jena.thur.de
  38  * have a look into http-anon.c to get more informations.
  39  */
  40
  41 #include "squid.h"
  42
  43 static const char *const crlf = "\r\n";
  44
  45 static CNCB httpConnectDone;
  46 static CWCB httpSendComplete;
  47 static CWCB httpSendRequestEntry;
  48
  49 static PF httpReadReply;
  50 static PF httpSendRequest;
  51 static PF httpStateFree;
  52 static PF httpTimeout;
  53 static void httpCacheNegatively(StoreEntry *);
  54 static void httpMakePrivate(StoreEntry *);
  55 static void httpMakePublic(StoreEntry *);
  56 static int httpCachableReply(HttpStateData *);
  57 static void httpMaybeRemovePublic(StoreEntry *, http_status);
  58
  59 static void
  60 httpStateFree(int fdnotused, void *data)
  61 {
  62     HttpStateData *httpState = data;
  63     if (httpState == NULL)
  64         return;
  65     storeUnlockObject(httpState->entry);
  66     if (httpState->reply_hdr) {
  67         memFree(MEM_8K_BUF, httpState->reply_hdr);
  68         httpState->reply_hdr = NULL;
  69     }
  70     requestUnlink(httpState->request);
  71     requestUnlink(httpState->orig_request);
  72     httpState->request = NULL;
  73     httpState->orig_request = NULL;
  74     cbdataFree(httpState);
  75 }
  76
  77 int
  78 httpCachable(method_t method)
  79 {
  80     /* GET and HEAD are cachable. Others are not. */
  81     if (method != METHOD_GET && method != METHOD_HEAD)
  82         return 0;
  83     /* else cachable */
  84     return 1;
  85 }
  86
  87 static void
  88 httpTimeout(int fd, void *data)
  89 {
  90     HttpStateData *httpState = data;
  91     StoreEntry *entry = httpState->entry;
  92     ErrorState *err;
  93     debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry));
  94     assert(entry->store_status == STORE_PENDING);
  95     if (entry->mem_obj->inmem_hi == 0) {
  96         err = errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT);
  97         err->request = requestLink(httpState->orig_request);
  98         errorAppendEntry(entry, err);
  99     } else {
 100         storeAbort(entry, 0);
 101     }
 102     comm_close(fd);
 103 }
 104
 105 /* This object can be cached for a long time */
 106 static void
 107 httpMakePublic(StoreEntry * entry)
 108 {
 109     if (entry->flags.entry_cachable)
 110         storeSetPublicKey(entry);
 111 }
 112
 113 /* This object should never be cached at all */
 114 static void
 115 httpMakePrivate(StoreEntry * entry)
 116 {
 117     storeExpireNow(entry);
 118     storeReleaseRequest(entry); /* delete object when not used */
 119     /* storeReleaseRequest clears ENTRY_CACHABLE flag */
 120 }
 121
 122 /* This object may be negatively cached */
 123 static void
 124 httpCacheNegatively(StoreEntry * entry)
 125 {
 126     storeNegativeCache(entry);
 127     if (entry->flags.entry_cachable)
 128         storeSetPublicKey(entry);
 129 }
 130
 131 static void
 132 httpMaybeRemovePublic(StoreEntry * e, http_status status)
 133 {
 134     int remove = 0;
 135     const cache_key *key;
 136     StoreEntry *pe;
 137     if (!e->flags.key_private)
 138         return;
 139     switch (status) {
 140     case HTTP_OK:
 141     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 142     case HTTP_MULTIPLE_CHOICES:
 143     case HTTP_MOVED_PERMANENTLY:
 144     case HTTP_MOVED_TEMPORARILY:
 145     case HTTP_FORBIDDEN:
 146     case HTTP_NOT_FOUND:
 147     case HTTP_METHOD_NOT_ALLOWED:
 148     case HTTP_GONE:
 149         remove = 1;
 150         break;
 151 #if WORK_IN_PROGRESS
 152     case HTTP_UNAUTHORIZED:
 153         remove = 1;
 154         break;
 155 #endif
 156     default:
 157         remove = 0;
 158         break;
 159     }
 160     if (!remove)
 161         return;
 162     assert(e->mem_obj);
 163     key = storeKeyPublic(e->mem_obj->url, e->mem_obj->method);
 164     if ((pe = storeGet(key)) != NULL) {
 165         assert(e != pe);
 166         storeRelease(pe);
 167     }
 168     if (e->mem_obj->method == METHOD_GET) {
 169         /* A fresh GET should eject old HEAD objects */
 170         key = storeKeyPublic(e->mem_obj->url, METHOD_HEAD);
 171         if ((pe = storeGet(key)) != NULL) {
 172             assert(e != pe);
 173             storeRelease(pe);
 174         }
 175     }
 176 }
 177
 178 static int
 179 httpCachableReply(HttpStateData * httpState)
 180 {
 181     HttpReply *rep = httpState->entry->mem_obj->reply;
 182     HttpHeader *hdr = &rep->header;
 183     const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0;
 184     if (EBIT_TEST(cc_mask, CC_PRIVATE))
 185         return 0;
 186     if (EBIT_TEST(cc_mask, CC_NO_CACHE))
 187         return 0;
 188     if (EBIT_TEST(cc_mask, CC_NO_STORE))
 189         return 0;
 190     if (httpState->request->flags.auth) {
 191         /*
 192          * Responses to requests with authorization may be cached
 193          * only if a Cache-Control: public reply header is present.
 194          * RFC 2068, sec 14.9.4
 195          */
 196         if (!EBIT_TEST(cc_mask, CC_PUBLIC))
 197             return 0;
 198     }
 199     /*
 200      * We don't properly deal with Vary features yet, so we can't
 201      * cache these
 202      */
 203     if (httpHeaderHas(hdr, HDR_VARY))
 204         return 0;
 205     switch (httpState->entry->mem_obj->reply->sline.status) {
 206         /* Responses that are cacheable */
 207     case HTTP_OK:
 208     case HTTP_NON_AUTHORITATIVE_INFORMATION:
 209     case HTTP_MULTIPLE_CHOICES:
 210     case HTTP_MOVED_PERMANENTLY:
 211     case HTTP_GONE:
 212         /* don't cache objects from peers w/o LMT, Date, or Expires */
 213         /* check that is it enough to check headers @?@ */
 214         if (rep->date > -1)
 215             return 1;
 216         else if (rep->last_modified > -1)
 217             return 1;
 218         else if (!httpState->peer)
 219             return 1;
 220         /* @?@ (here and 302): invalid expires header compiles to squid_curtime */
 221         else if (rep->expires > -1)
 222             return 1;
 223         else
 224             return 0;
 225         /* NOTREACHED */
 226         break;
 227         /* Responses that only are cacheable if the server says so */
 228     case HTTP_MOVED_TEMPORARILY:
 229         if (rep->expires > -1)
 230             return 1;
 231         else
 232             return 0;
 233         /* NOTREACHED */
 234         break;
 235         /* Errors can be negatively cached */
 236     case HTTP_NO_CONTENT:
 237     case HTTP_USE_PROXY:
 238     case HTTP_BAD_REQUEST:
 239     case HTTP_FORBIDDEN:
 240     case HTTP_NOT_FOUND:
 241     case HTTP_METHOD_NOT_ALLOWED:
 242     case HTTP_REQUEST_URI_TOO_LARGE:
 243     case HTTP_INTERNAL_SERVER_ERROR:
 244     case HTTP_NOT_IMPLEMENTED:
 245     case HTTP_BAD_GATEWAY:
 246     case HTTP_SERVICE_UNAVAILABLE:
 247     case HTTP_GATEWAY_TIMEOUT:
 248         return -1;
 249         /* NOTREACHED */
 250         break;
 251         /* Some responses can never be cached */
 252     case HTTP_PARTIAL_CONTENT:  /* Not yet supported */
 253     case HTTP_SEE_OTHER:
 254     case HTTP_NOT_MODIFIED:
 255     case HTTP_UNAUTHORIZED:
 256     case HTTP_PROXY_AUTHENTICATION_REQUIRED:
 257     case HTTP_INVALID_HEADER:   /* Squid header parsing error */
 258     default:                    /* Unknown status code */
 259         return 0;
 260         /* NOTREACHED */
 261         break;
 262     }
 263     /* NOTREACHED */
 264 }
 265
 266 /* rewrite this later using new interfaces @?@ */
 267 void
 268 httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size)
 269 {
 270     char *t = NULL;
 271     StoreEntry *entry = httpState->entry;
 272     int room;
 273     int hdr_len;
 274     HttpReply *reply = entry->mem_obj->reply;
 275     debug(11, 3) ("httpProcessReplyHeader: key '%s'\n",
 276         storeKeyText(entry->key));
 277     if (httpState->reply_hdr == NULL)
 278         httpState->reply_hdr = memAllocate(MEM_8K_BUF);
 279     if (httpState->reply_hdr_state == 0) {
 280         hdr_len = strlen(httpState->reply_hdr);
 281         room = 8191 - hdr_len;
 282         strncat(httpState->reply_hdr, buf, room < size ? room : size);
 283         hdr_len += room < size ? room : size;
 284         if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) {
 285             debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr);
 286             httpState->reply_hdr_state += 2;
 287             reply->sline.status = HTTP_INVALID_HEADER;
 288             return;
 289         }
 290         t = httpState->reply_hdr + hdr_len;
 291         /* headers can be incomplete only if object still arriving */
 292         if (!httpState->eof) {
 293             size_t k = headersEnd(httpState->reply_hdr, 8192);
 294             if (0 == k)
 295                 return;         /* headers not complete */
 296             t = httpState->reply_hdr + k;
 297         }
 298         *t = '\0';
 299         httpState->reply_hdr_state++;
 300     }
 301     if (httpState->reply_hdr_state == 1) {
 302         const Ctx ctx = ctx_enter(entry->mem_obj->url);
 303         httpState->reply_hdr_state++;
 304         debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n",
 305             httpState->reply_hdr);
 306         /* Parse headers into reply structure */
 307         /* what happens if we fail to parse here? */
 308         httpReplyParse(reply, httpState->reply_hdr);    /* httpState->eof); */
 309         storeTimestampsSet(entry);
 310         /* Check if object is cacheable or not based on reply code */
 311         debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status);
 312         if (neighbors_do_private_keys)
 313             httpMaybeRemovePublic(entry, reply->sline.status);
 314         switch (httpCachableReply(httpState)) {
 315         case 1:
 316             httpMakePublic(entry);
 317             break;
 318         case 0:
 319             httpMakePrivate(entry);
 320             break;
 321         case -1:
 322             httpCacheNegatively(entry);
 323             break;
 324         default:
 325             assert(0);
 326             break;
 327         }
 328         if (reply->cache_control) {
 329             if (EBIT_TEST(reply->cache_control->mask, CC_PROXY_REVALIDATE))
 330                 entry->flags.entry_revalidate = 1;
 331             else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE))
 332                 entry->flags.entry_revalidate = 1;
 333         }
 334         if (httpState->flags.keepalive)
 335             if (httpState->peer)
 336                 httpState->peer->stats.n_keepalives_sent++;
 337         if (reply->keep_alive)
 338             if (httpState->peer)
 339                 httpState->peer->stats.n_keepalives_recv++;
 340         ctx_exit(ctx);
 341     }
 342 }
 343
 344 static int
 345 httpPconnTransferDone(HttpStateData * httpState)
 346 {
 347     /* return 1 if we got the last of the data on a persistent connection */
 348     MemObject *mem = httpState->entry->mem_obj;
 349     HttpReply *reply = mem->reply;
 350     debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd);
 351     /*
 352      * If we didn't send a keep-alive request header, then this
 353      * can not be a persistent connection.
 354      */
 355     if (!httpState->flags.keepalive)
 356         return 0;
 357     /*
 358      * What does the reply have to say about keep-alive?
 359      */
 360     if (!reply->keep_alive)
 361         return 0;
 362     debug(11, 5) ("httpPconnTransferDone: content_length=%d\n",
 363         reply->content_length);
 364     /*
 365      * Deal with gross HTTP stuff
 366      *    - If we haven't seen the end of the reply headers, we can't
 367      *      be persistent.
 368      *    - For HEAD requests we're done.
 369      *    - For "200 OK" check the content-length in the next block.
 370      *    - For "204 No Content" (even with content-length) we're done.
 371      *    - For "304 Not Modified" (even with content-length) we're done.
 372      *    - 1XX replies never have a body; we're done.
 373      *    - For all other replies, check content length in next block.
 374      */
 375     if (httpState->reply_hdr_state < 2)
 376         return 0;
 377     else if (httpState->request->method == METHOD_HEAD)
 378         return 1;
 379     else if (reply->sline.status == HTTP_OK)
 380         (void) 0;               /* common case, continue */
 381     else if (reply->sline.status == HTTP_NO_CONTENT)
 382         return 1;
 383     else if (reply->sline.status == HTTP_NOT_MODIFIED)
 384         return 1;
 385     else if (reply->sline.status < HTTP_OK)
 386         return 1;
 387     /*
 388      * If there is no content-length, then we can't be
 389      * persistent.  If there is a content length, then we must
 390      * wait until we've seen the end of the body.
 391      */
 392     if (reply->content_length < 0)
 393         return 0;
 394     else if (mem->inmem_hi < reply->content_length + reply->hdr_sz)
 395         return 0;
 396     else
 397         return 1;
 398 }
 399
 400 /* This will be called when data is ready to be read from fd.  Read until
 401  * error or connection closed. */
 402 /* XXX this function is too long! */
 403 static void
 404 httpReadReply(int fd, void *data)
 405 {
 406     HttpStateData *httpState = data;
 407     LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF);
 408     StoreEntry *entry = httpState->entry;
 409     const request_t *request = httpState->request;
 410     int len;
 411     int bin;
 412     int clen;
 413     size_t read_sz;
 414 #if DELAY_POOLS
 415     delay_id delay_id = delayMostBytesAllowed(entry->mem_obj);
 416 #endif
 417     if (fwdAbortFetch(entry)) {
 418         storeAbort(entry, 0);
 419         comm_close(fd);
 420         return;
 421     }
 422     /* check if we want to defer reading */
 423     errno = 0;
 424     read_sz = SQUID_TCP_SO_RCVBUF;
 425 #if DELAY_POOLS
 426     read_sz = delayBytesWanted(delay_id, 1, read_sz);
 427 #endif
 428     Counter.syscalls.sock.reads++;
 429     len = read(fd, buf, read_sz);
 430     debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len);
 431     if (len > 0) {
 432         fd_bytes(fd, len, FD_READ);
 433 #if DELAY_POOLS
 434         delayBytesIn(delay_id, len);
 435 #endif
 436         kb_incr(&Counter.server.all.kbytes_in, len);
 437         kb_incr(&Counter.server.http.kbytes_in, len);
 438         commSetTimeout(fd, Config.Timeout.read, NULL, NULL);
 439         IOStats.Http.reads++;
 440         for (clen = len - 1, bin = 0; clen; bin++)
 441             clen >>= 1;
 442         IOStats.Http.read_hist[bin]++;
 443     }
 444     if (!httpState->reply_hdr && len > 0) {
 445         /* Skip whitespace */
 446         while (len > 0 && isspace(*buf))
 447             xmemmove(buf, buf + 1, len--);
 448         if (len == 0) {
 449             /* Continue to read... */
 450             commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
 451             return;
 452         }
 453     }
 454     if (len < 0) {
 455         debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n",
 456             fd, xstrerror());
 457         if (ignoreErrno(errno)) {
 458             commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
 459         } else if (entry->mem_obj->inmem_hi == 0) {
 460             fwdFail(httpState->fwdState, ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR, errno);
 461             comm_close(fd);
 462         } else {
 463             storeAbort(entry, 0);
 464             comm_close(fd);
 465         }
 466     } else if (len == 0 && entry->mem_obj->inmem_hi == 0) {
 467         fwdFail(httpState->fwdState, ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE, errno);
 468         httpState->eof = 1;
 469         comm_close(fd);
 470     } else if (len == 0) {
 471         /* Connection closed; retrieval done. */
 472         httpState->eof = 1;
 473         if (httpState->reply_hdr_state < 2)
 474             /*
 475              * Yes Henrik, there is a point to doing this.  When we
 476              * called httpProcessReplyHeader() before, we didn't find
 477              * the end of headers, but now we are definately at EOF, so
 478              * we want to process the reply headers.
 479              */
 480             httpProcessReplyHeader(httpState, buf, len);
 481 #ifdef PPNR_WIP
 482         storePPNR(entry);
 483 #endif /* PPNR_WIP */
 484         storeComplete(entry);   /* deallocates mem_obj->request */
 485         comm_close(fd);
 486     } else {
 487 #ifndef PPNR_WIP
 488         if (httpState->reply_hdr_state < 2)
 489 #else
 490         if (httpState->reply_hdr_state < 2) {
 491 #endif /* PPNR_WIP */
 492             httpProcessReplyHeader(httpState, buf, len);
 493 #ifdef PPNR_WIP
 494             if (httpState->reply_hdr_state == 2)
 495                 storePPNR(entry);
 496         }
 497 #endif /* PPNR_WIP */
 498         storeAppend(entry, buf, len);
 499 #ifdef OPTIMISTIC_IO
 500         if (entry->store_status == STORE_ABORTED) {
 501             /*
 502              * the above storeAppend() call could ABORT this entry,
 503              * in that case, the server FD should already be closed.
 504              * there's nothing for us to do.
 505              */
 506             (void) 0;
 507         } else
 508 #endif
 509         if (httpPconnTransferDone(httpState)) {
 510             /* yes we have to clear all these! */
 511             commSetDefer(fd, NULL, NULL);
 512             commSetTimeout(fd, -1, NULL, NULL);
 513             commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0);
 514             comm_remove_close_handler(fd, httpStateFree, httpState);
 515             storeComplete(entry);       /* deallocates mem_obj->request */
 516             /* call storeComplete BEFORE fwdUnregister or else fwdUnregister
 517              * will storeAbort */
 518             fwdUnregister(fd, httpState->fwdState);
 519             pconnPush(fd, request->host, request->port);
 520             httpState->fd = -1;
 521             httpStateFree(-1, httpState);
 522         } else {
 523             /* Wait for EOF condition */
 524             commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0);
 525         }
 526     }
 527 }
 528
 529 /* This will be called when request write is complete. Schedule read of
 530  * reply. */
 531 static void
 532 httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data)
 533 {
 534     HttpStateData *httpState = data;
 535     StoreEntry *entry = httpState->entry;
 536     ErrorState *err;
 537     debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n",
 538         fd, size, errflag);
 539     if (size > 0) {
 540         fd_bytes(fd, size, FD_WRITE);
 541         kb_incr(&Counter.server.all.kbytes_out, size);
 542         kb_incr(&Counter.server.http.kbytes_out, size);
 543     }
 544     if (errflag == COMM_ERR_CLOSING)
 545         return;
 546     if (errflag) {
 547         err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR);
 548         err->xerrno = errno;
 549         err->request = requestLink(httpState->orig_request);
 550         errorAppendEntry(entry, err);
 551         comm_close(fd);
 552         return;
 553     } else {
 554         /* Schedule read reply. */
 555         commSetSelect(fd,
 556             COMM_SELECT_READ,
 557             httpReadReply,
 558             httpState, 0);
 559         commSetDefer(fd, fwdCheckDeferRead, entry);
 560     }
 561 }
 562
 563 /*
 564  * build request headers and append them to a given MemBuf
 565  * used by httpBuildRequestPrefix()
 566  * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing
 567  */
 568 void
 569 httpBuildRequestHeader(request_t * request,
 570     request_t * orig_request,
 571     StoreEntry * entry,
 572     HttpHeader * hdr_out,
 573     int cfd,
 574     http_state_flags flags)
 575 {
 576     /* building buffer for complex strings */
 577 #define BBUF_SZ (MAX_URL+32)
 578     LOCAL_ARRAY(char, bbuf, BBUF_SZ);
 579     String strConnection = StringNull;
 580     const HttpHeader *hdr_in = &orig_request->header;
 581     int filter_range;
 582     const HttpHeaderEntry *e;
 583     HttpHeaderPos pos = HttpHeaderInitPos;
 584     httpHeaderInit(hdr_out, hoRequest);
 585     /* append our IMS header */
 586     if (entry && entry->lastmod > -1 && request->method == METHOD_GET)
 587         httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, entry->lastmod);
 588
 589     /* decide if we want to filter out Range specs
 590      * no reason to filter out if the reply will not be cachable
 591      * or if we cannot parse the specs */
 592     filter_range =
 593         orig_request->range && orig_request->flags.cachable;
 594
 595     strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION);
 596     while ((e = httpHeaderGetEntry(hdr_in, &pos))) {
 597         debug(11, 5) ("httpBuildRequestHeader: %s: %s\n",
 598             strBuf(e->name), strBuf(e->value));
 599         if (!httpRequestHdrAllowed(e, &strConnection))
 600             continue;
 601         switch (e->id) {
 602         case HDR_PROXY_AUTHORIZATION:
 603             /* If we're not going to do proxy auth, then it must be passed on */
 604             if (!request->flags.used_proxy_auth)
 605                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
 606             break;
 607         case HDR_HOST:
 608             /* Don't use client's Host: header for redirected requests */
 609             if (!request->flags.redirected)
 610                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
 611             break;
 612         case HDR_IF_MODIFIED_SINCE:
 613             /* append unless we added our own;
 614              * note: at most one client's ims header can pass through */
 615             if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE))
 616                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
 617             break;
 618         case HDR_MAX_FORWARDS:
 619             if (orig_request->method == METHOD_TRACE) {
 620                 /* sacrificing efficiency over clarity, etc. */
 621                 const int hops = httpHeaderGetInt(hdr_in, HDR_MAX_FORWARDS);
 622                 if (hops > 0)
 623                     httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1);
 624             }
 625             break;
 626         case HDR_RANGE:
 627         case HDR_IF_RANGE:
 628             if (!filter_range)
 629                 httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
 630             break;
 631         case HDR_PROXY_CONNECTION:
 632         case HDR_CONNECTION:
 633         case HDR_VIA:
 634         case HDR_X_FORWARDED_FOR:
 635         case HDR_CACHE_CONTROL:
 636             /* append these after the loop if needed */
 637             break;
 638         default:
 639             /* pass on all other header fields */
 640             httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e));
 641         }
 642     }
 643
 644     /* append fake user agent if configured and
 645      * the real one is not supplied by the client */
 646     if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT))
 647         httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua);
 648
 649     /* append Via */
 650     {
 651         String strVia = httpHeaderGetList(hdr_in, HDR_VIA);
 652         snprintf(bbuf, BBUF_SZ, "%3.1f %s", orig_request->http_ver, ThisCache);
 653         strListAdd(&strVia, bbuf, ',');
 654         httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia));
 655         stringClean(&strVia);
 656     }
 657     /* append X-Forwarded-For */
 658     {
 659         String strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR);
 660         strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ',');
 661         httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd));
 662         stringClean(&strFwd);
 663     }
 664     /* append Host if not there already */
 665     if (!httpHeaderHas(hdr_out, HDR_HOST)) {
 666         /* use port# only if not default */
 667         if (orig_request->port == urlDefaultPort(orig_request->protocol)) {
 668             httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host);
 669         } else {
 670             httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d",
 671                 orig_request->host, (int) orig_request->port);
 672         }
 673     }
 674     /* append Cache-Control, add max-age if not there already */
 675     {
 676         HttpHdrCc *cc = httpHeaderGetCc(hdr_in);
 677         if (!cc)
 678             cc = httpHdrCcCreate();
 679         if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) {
 680             const char *url = entry ? storeUrl(entry) : urlCanonical(orig_request);
 681             httpHdrCcSetMaxAge(cc, getMaxAge(url));
 682             if (strLen(request->urlpath))
 683                 assert(strstr(url, strBuf(request->urlpath)));
 684         }
 685         httpHeaderPutCc(hdr_out, cc);
 686         httpHdrCcDestroy(cc);
 687     }
 688     /* maybe append Connection: keep-alive */
 689     if (flags.keepalive) {
 690         if (flags.proxying) {
 691             httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive");
 692         } else {
 693             httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive");
 694         }
 695     }
 696     stringClean(&strConnection);
 697 }
 698
 699 /* build request prefix and append it to a given MemBuf;
 700  * return the length of the prefix */
 701 size_t
 702 httpBuildRequestPrefix(request_t * request,
 703     request_t * orig_request,
 704     StoreEntry * entry,
 705     MemBuf * mb,
 706     int cfd,
 707     http_state_flags flags)
 708 {
 709     const int offset = mb->size;
 710     memBufPrintf(mb, "%s %s HTTP/1.0\r\n",
 711         RequestMethodStr[request->method],
 712         strLen(request->urlpath) ? strBuf(request->urlpath) : "/");
 713     /* build and pack headers */
 714     {
 715         HttpHeader hdr;
 716         Packer p;
 717         httpBuildRequestHeader(request, orig_request, entry, &hdr, cfd, flags);
 718         packerToMemInit(&p, mb);
 719         httpHeaderPackInto(&hdr, &p);
 720         httpHeaderClean(&hdr);
 721         packerClean(&p);
 722     }
 723     /* append header terminator */
 724     memBufAppend(mb, "\r\n", 2);
 725     return mb->size - offset;
 726 }
 727
 728 /* This will be called when connect completes. Write request. */
 729 static void
 730 httpSendRequest(int fd, void *data)
 731 {
 732     HttpStateData *httpState = data;
 733     MemBuf mb;
 734     request_t *req = httpState->request;
 735     StoreEntry *entry = httpState->entry;
 736     int cfd;
 737     peer *p = httpState->peer;
 738     CWCB *sendHeaderDone;
 739
 740     debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", fd, httpState);
 741
 742     if (pumpMethod(req->method))
 743         sendHeaderDone = httpSendRequestEntry;
 744     else
 745         sendHeaderDone = httpSendComplete;
 746
 747     if (!opt_forwarded_for)
 748         cfd = -1;
 749     else if (entry->mem_obj == NULL)
 750         cfd = -1;
 751     else
 752         cfd = entry->mem_obj->fd;
 753     assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type);
 754     if (p != NULL)
 755         httpState->flags.proxying = 1;
 756     /*
 757      * Is keep-alive okay for all request methods?
 758      */
 759     if (p == NULL)
 760         httpState->flags.keepalive = 1;
 761     else if (p->stats.n_keepalives_sent < 10)
 762         httpState->flags.keepalive = 1;
 763     else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50)
 764         httpState->flags.keepalive = 1;
 765     memBufDefInit(&mb);
 766     httpBuildRequestPrefix(req,
 767         httpState->orig_request,
 768         entry,
 769         &mb,
 770         cfd,
 771         httpState->flags);
 772     debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", fd, mb.buf);
 773     comm_write_mbuf(fd, mb, sendHeaderDone, httpState);
 774 }
 775
 776 void
 777 httpStart(FwdState * fwdState, int fd)
 778 {
 779     HttpStateData *httpState = memAllocate(MEM_HTTP_STATE_DATA);
 780     request_t *proxy_req;
 781     request_t *orig_req = fwdState->request;
 782     debug(11, 3) ("httpStart: \"%s %s\"\n",
 783         RequestMethodStr[orig_req->method],
 784         storeUrl(fwdState->entry));
 785     cbdataAdd(httpState, MEM_HTTP_STATE_DATA);
 786     storeLockObject(fwdState->entry);
 787     httpState->fwdState = fwdState;
 788     httpState->entry = fwdState->entry;
 789     httpState->fd = fd;
 790     if (fwdState->servers)
 791         httpState->peer = fwdState->servers->peer;      /* might be NULL */
 792     if (httpState->peer) {
 793         proxy_req = requestCreate(orig_req->method,
 794             PROTO_NONE, storeUrl(httpState->entry));
 795         xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN);
 796         proxy_req->port = httpState->peer->http_port;
 797         proxy_req->flags = orig_req->flags;
 798         httpState->request = requestLink(proxy_req);
 799         httpState->orig_request = requestLink(orig_req);
 800         proxy_req->flags.proxying = 1;
 801         /*
 802          * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
 803          * We might end up getting the object from somewhere else if,
 804          * for example, the request to this neighbor fails.
 805          */
 806         if (httpState->peer->options.proxy_only)
 807             storeReleaseRequest(httpState->entry);
 808 #if DELAY_POOLS
 809         if (httpState->peer->options.no_delay) {
 810             proxy_req->delay_id = 0;
 811         } else {
 812             proxy_req->delay_id = orig_req->delay_id;
 813         }
 814 #endif
 815     } else {
 816         httpState->request = requestLink(orig_req);
 817         httpState->orig_request = requestLink(orig_req);
 818     }
 819     /*
 820      * register the handler to free HTTP state data when the FD closes
 821      */
 822     comm_add_close_handler(fd, httpStateFree, httpState);
 823     Counter.server.all.requests++;
 824     Counter.server.http.requests++;
 825     httpConnectDone(fd, COMM_OK, httpState);
 826 }
 827
 828 static void
 829 httpConnectDone(int fd, int status, void *data)
 830 {
 831     HttpStateData *httpState = data;
 832     request_t *request = httpState->request;
 833     StoreEntry *entry = httpState->entry;
 834     ErrorState *err;
 835     if (status == COMM_ERR_DNS) {
 836         debug(11, 4) ("httpConnectDone: Unknown host: %s\n", request->host);
 837         err = errorCon(ERR_DNS_FAIL, HTTP_SERVICE_UNAVAILABLE);
 838         err->dnsserver_msg = xstrdup(dns_error_message);
 839         err->request = requestLink(httpState->orig_request);
 840         errorAppendEntry(entry, err);
 841         comm_close(fd);
 842     } else if (status != COMM_OK) {
 843         err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE);
 844         err->xerrno = errno;
 845         err->host = xstrdup(request->host);
 846         err->port = request->port;
 847         err->request = requestLink(httpState->orig_request);
 848         errorAppendEntry(entry, err);
 849         if (httpState->peer)
 850             peerCheckConnectStart(httpState->peer);
 851         comm_close(fd);
 852     } else {
 853         commSetSelect(fd, COMM_SELECT_WRITE, httpSendRequest, httpState, 0);
 854         commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState);
 855     }
 856 }
 857
 858 static void
 859 httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void *data)
 860 {
 861     HttpStateData *httpState = data;
 862     StoreEntry *entry = httpState->entry;
 863     ErrorState *err;
 864     debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n",
 865         fd, size, errflag);
 866     if (size > 0) {
 867         fd_bytes(fd, size, FD_WRITE);
 868         kb_incr(&Counter.server.all.kbytes_out, size);
 869         kb_incr(&Counter.server.http.kbytes_out, size);
 870     }
 871     if (errflag == COMM_ERR_CLOSING)
 872         return;
 873     if (errflag) {
 874         err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR);
 875         err->xerrno = errno;
 876         err->request = requestLink(httpState->orig_request);
 877         errorAppendEntry(entry, err);
 878         comm_close(fd);
 879         return;
 880     }
 881     pumpStart(fd, entry, httpState->orig_request, httpSendComplete, httpState);
 882 }