]>
Commit | Line | Data |
---|---|---|
da2b3a17 | 1 | |
30a4f2a8 | 2 | /* |
447e176b | 3 | * $Id: http.cc,v 1.305 1998/08/14 09:22:37 wessels Exp $ |
30a4f2a8 | 4 | * |
5 | * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) | |
6 | * AUTHOR: Harvest Derived | |
7 | * | |
42c04c16 | 8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
e25c139f | 9 | * ---------------------------------------------------------- |
30a4f2a8 | 10 | * |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
e25c139f | 13 | * National Laboratory for Applied Network Research and funded by the |
14 | * National Science Foundation. Squid is Copyrighted (C) 1998 by | |
15 | * Duane Wessels and the University of California San Diego. Please | |
16 | * see the COPYRIGHT file for full details. Squid incorporates | |
17 | * software developed and/or copyrighted by other sources. Please see | |
18 | * the CREDITS file for full details. | |
30a4f2a8 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
cbdec147 | 32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. |
e25c139f | 33 | * |
30a4f2a8 | 34 | */ |
019dd986 | 35 | |
4a83b852 | 36 | /* |
37 | * Anonymizing patch by lutz@as-node.jena.thur.de | |
de3bdb4c | 38 | * have a look into http-anon.c to get more informations. |
4a83b852 | 39 | */ |
40 | ||
44a47c6e | 41 | #include "squid.h" |
090089c4 | 42 | |
6bf8443a | 43 | static const char *const crlf = "\r\n"; |
4db43fab | 44 | |
9e4ad609 | 45 | static CNCB httpConnectDone; |
46 | static CWCB httpSendComplete; | |
54220df8 | 47 | static CWCB httpSendRequestEntry; |
48 | ||
9e4ad609 | 49 | static PF httpReadReply; |
50 | static PF httpSendRequest; | |
51 | static PF httpStateFree; | |
52 | static PF httpTimeout; | |
f5b8bbc4 | 53 | static void httpCacheNegatively(StoreEntry *); |
54 | static void httpMakePrivate(StoreEntry *); | |
55 | static void httpMakePublic(StoreEntry *); | |
f8309b15 | 56 | static int httpCachableReply(HttpStateData *); |
b8d8561b | 57 | |
b177367b | 58 | static void |
79d39a72 | 59 | httpStateFree(int fdnotused, void *data) |
f5558c95 | 60 | { |
b177367b | 61 | HttpStateData *httpState = data; |
0d4d4170 | 62 | if (httpState == NULL) |
b177367b | 63 | return; |
f88211e8 | 64 | storeUnlockObject(httpState->entry); |
0d4d4170 | 65 | if (httpState->reply_hdr) { |
3f6c0fb2 | 66 | memFree(MEM_8K_BUF, httpState->reply_hdr); |
0d4d4170 | 67 | httpState->reply_hdr = NULL; |
68 | } | |
30a4f2a8 | 69 | requestUnlink(httpState->request); |
20cc1450 | 70 | requestUnlink(httpState->orig_request); |
7dd44885 | 71 | httpState->request = NULL; |
72 | httpState->orig_request = NULL; | |
73 | cbdataFree(httpState); | |
f5558c95 | 74 | } |
75 | ||
b8d8561b | 76 | int |
75e88d56 | 77 | httpCachable(method_t method) |
090089c4 | 78 | { |
090089c4 | 79 | /* GET and HEAD are cachable. Others are not. */ |
6eb42cae | 80 | if (method != METHOD_GET && method != METHOD_HEAD) |
090089c4 | 81 | return 0; |
090089c4 | 82 | /* else cachable */ |
83 | return 1; | |
84 | } | |
85 | ||
b8d8561b | 86 | static void |
5c5783a2 | 87 | httpTimeout(int fd, void *data) |
090089c4 | 88 | { |
b177367b | 89 | HttpStateData *httpState = data; |
593c9a75 | 90 | StoreEntry *entry = httpState->entry; |
9b312a19 | 91 | ErrorState *err; |
9fb13bb6 | 92 | debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); |
8796b9e9 | 93 | assert(entry->store_status == STORE_PENDING); |
73a3014d | 94 | if (entry->mem_obj->inmem_hi == 0) { |
fe40a877 | 95 | err = errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT); |
79a15e0a | 96 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 97 | errorAppendEntry(entry, err); |
b50179a6 | 98 | } else { |
b34ed725 | 99 | storeAbort(entry, 0); |
9b312a19 | 100 | } |
0d4d4170 | 101 | comm_close(fd); |
090089c4 | 102 | } |
103 | ||
30a4f2a8 | 104 | /* This object can be cached for a long time */ |
b8d8561b | 105 | static void |
106 | httpMakePublic(StoreEntry * entry) | |
30a4f2a8 | 107 | { |
79a15e0a | 108 | if (EBIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 109 | storeSetPublicKey(entry); |
110 | } | |
111 | ||
112 | /* This object should never be cached at all */ | |
b8d8561b | 113 | static void |
114 | httpMakePrivate(StoreEntry * entry) | |
30a4f2a8 | 115 | { |
30a4f2a8 | 116 | storeExpireNow(entry); |
79a15e0a | 117 | EBIT_CLR(entry->flag, ENTRY_CACHABLE); |
30a4f2a8 | 118 | storeReleaseRequest(entry); /* delete object when not used */ |
119 | } | |
120 | ||
121 | /* This object may be negatively cached */ | |
b8d8561b | 122 | static void |
123 | httpCacheNegatively(StoreEntry * entry) | |
30a4f2a8 | 124 | { |
79b5cc5f | 125 | storeNegativeCache(entry); |
79a15e0a | 126 | if (EBIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 127 | storeSetPublicKey(entry); |
30a4f2a8 | 128 | } |
129 | ||
f8309b15 | 130 | static int |
131 | httpCachableReply(HttpStateData * httpState) | |
c54e9052 | 132 | { |
d8b249ef | 133 | HttpReply *rep = httpState->entry->mem_obj->reply; |
134 | HttpHeader *hdr = &rep->header; | |
135 | const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0; | |
7faf2bdb | 136 | if (EBIT_TEST(cc_mask, CC_PRIVATE)) |
f8309b15 | 137 | return 0; |
7faf2bdb | 138 | if (EBIT_TEST(cc_mask, CC_NO_CACHE)) |
f8309b15 | 139 | return 0; |
ed2f05a1 | 140 | if (EBIT_TEST(cc_mask, CC_NO_STORE)) |
141 | return 0; | |
a6dfe2d9 | 142 | if (EBIT_TEST(httpState->request->flags, REQ_AUTH)) { |
143 | /* | |
144 | * Responses to requests with authorization may be cached | |
68aefb7d | 145 | * only if a Cache-Control: public reply header is present. |
a6dfe2d9 | 146 | * RFC 2068, sec 14.9.4 |
147 | */ | |
148 | if (!EBIT_TEST(cc_mask, CC_PUBLIC)) | |
fee0cebb | 149 | return 0; |
a6dfe2d9 | 150 | } |
f8309b15 | 151 | /* |
02fe0fbc | 152 | * We don't properly deal with Vary features yet, so we can't |
153 | * cache these | |
f8309b15 | 154 | */ |
783e4699 | 155 | if (httpHeaderHas(hdr, HDR_VARY)) |
156 | return 0; | |
cb69b4c7 | 157 | switch (httpState->entry->mem_obj->reply->sline.status) { |
c54e9052 | 158 | /* Responses that are cacheable */ |
19a04dac | 159 | case HTTP_OK: |
160 | case HTTP_NON_AUTHORITATIVE_INFORMATION: | |
161 | case HTTP_MULTIPLE_CHOICES: | |
162 | case HTTP_MOVED_PERMANENTLY: | |
163 | case HTTP_GONE: | |
1294c0fc | 164 | /* don't cache objects from peers w/o LMT, Date, or Expires */ |
cb69b4c7 | 165 | /* check that is it enough to check headers @?@ */ |
d8b249ef | 166 | if (rep->date > -1) |
c54e9052 | 167 | return 1; |
d8b249ef | 168 | else if (rep->last_modified > -1) |
c54e9052 | 169 | return 1; |
1294c0fc | 170 | else if (!httpState->peer) |
c54e9052 | 171 | return 1; |
d8b249ef | 172 | /* @?@ (here and 302): invalid expires header compiles to squid_curtime */ |
173 | else if (rep->expires > -1) | |
c54e9052 | 174 | return 1; |
c54e9052 | 175 | else |
176 | return 0; | |
79d39a72 | 177 | /* NOTREACHED */ |
c54e9052 | 178 | break; |
179 | /* Responses that only are cacheable if the server says so */ | |
19a04dac | 180 | case HTTP_MOVED_TEMPORARILY: |
d8b249ef | 181 | if (rep->expires > -1) |
c54e9052 | 182 | return 1; |
183 | else | |
184 | return 0; | |
79d39a72 | 185 | /* NOTREACHED */ |
c54e9052 | 186 | break; |
187 | /* Errors can be negatively cached */ | |
19a04dac | 188 | case HTTP_NO_CONTENT: |
189 | case HTTP_USE_PROXY: | |
190 | case HTTP_BAD_REQUEST: | |
191 | case HTTP_FORBIDDEN: | |
192 | case HTTP_NOT_FOUND: | |
193 | case HTTP_METHOD_NOT_ALLOWED: | |
194 | case HTTP_REQUEST_URI_TOO_LARGE: | |
195 | case HTTP_INTERNAL_SERVER_ERROR: | |
196 | case HTTP_NOT_IMPLEMENTED: | |
197 | case HTTP_BAD_GATEWAY: | |
198 | case HTTP_SERVICE_UNAVAILABLE: | |
199 | case HTTP_GATEWAY_TIMEOUT: | |
c54e9052 | 200 | return -1; |
79d39a72 | 201 | /* NOTREACHED */ |
c54e9052 | 202 | break; |
203 | /* Some responses can never be cached */ | |
0cdcddb9 | 204 | case HTTP_PARTIAL_CONTENT: /* Not yet supported */ |
19a04dac | 205 | case HTTP_SEE_OTHER: |
206 | case HTTP_NOT_MODIFIED: | |
207 | case HTTP_UNAUTHORIZED: | |
208 | case HTTP_PROXY_AUTHENTICATION_REQUIRED: | |
0cdcddb9 | 209 | case HTTP_INVALID_HEADER: /* Squid header parsing error */ |
c54e9052 | 210 | default: /* Unknown status code */ |
211 | return 0; | |
79d39a72 | 212 | /* NOTREACHED */ |
c54e9052 | 213 | break; |
214 | } | |
79d39a72 | 215 | /* NOTREACHED */ |
c54e9052 | 216 | } |
090089c4 | 217 | |
cb69b4c7 | 218 | /* rewrite this later using new interfaces @?@ */ |
b8d8561b | 219 | void |
0ee4272b | 220 | httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) |
f5558c95 | 221 | { |
222 | char *t = NULL; | |
30a4f2a8 | 223 | StoreEntry *entry = httpState->entry; |
d3fb4dea | 224 | int room; |
225 | int hdr_len; | |
cb69b4c7 | 226 | HttpReply *reply = entry->mem_obj->reply; |
b6cfb65c | 227 | debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", |
228 | storeKeyText(entry->key)); | |
e924600d | 229 | if (httpState->reply_hdr == NULL) |
7021844c | 230 | httpState->reply_hdr = memAllocate(MEM_8K_BUF); |
30a4f2a8 | 231 | if (httpState->reply_hdr_state == 0) { |
232 | hdr_len = strlen(httpState->reply_hdr); | |
ed85b771 | 233 | room = 8191 - hdr_len; |
30a4f2a8 | 234 | strncat(httpState->reply_hdr, buf, room < size ? room : size); |
d3fb4dea | 235 | hdr_len += room < size ? room : size; |
30a4f2a8 | 236 | if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { |
84fa351c | 237 | debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); |
30a4f2a8 | 238 | httpState->reply_hdr_state += 2; |
cb69b4c7 | 239 | reply->sline.status = 555; |
ed85b771 | 240 | return; |
d3fb4dea | 241 | } |
d1a43e28 | 242 | t = httpState->reply_hdr + hdr_len; |
243 | /* headers can be incomplete only if object still arriving */ | |
2334c194 | 244 | if (!httpState->eof) { |
245 | size_t k = headersEnd(httpState->reply_hdr, 8192); | |
246 | if (0 == k) | |
d1a43e28 | 247 | return; /* headers not complete */ |
2334c194 | 248 | t = httpState->reply_hdr + k; |
249 | } | |
2285407f | 250 | *t = '\0'; |
30a4f2a8 | 251 | httpState->reply_hdr_state++; |
f5558c95 | 252 | } |
30a4f2a8 | 253 | if (httpState->reply_hdr_state == 1) { |
123abbe1 | 254 | const Ctx ctx = ctx_enter(entry->mem_obj->url); |
30a4f2a8 | 255 | httpState->reply_hdr_state++; |
a3d5953d | 256 | debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", |
30a4f2a8 | 257 | httpState->reply_hdr); |
258 | /* Parse headers into reply structure */ | |
2246b732 | 259 | /* what happens if we fail to parse here? */ |
ee1679df | 260 | httpReplyParse(reply, httpState->reply_hdr); /* httpState->eof); */ |
ca98227c | 261 | storeTimestampsSet(entry); |
30a4f2a8 | 262 | /* Check if object is cacheable or not based on reply code */ |
cb69b4c7 | 263 | debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); |
f8309b15 | 264 | switch (httpCachableReply(httpState)) { |
c54e9052 | 265 | case 1: |
266 | httpMakePublic(entry); | |
30a4f2a8 | 267 | break; |
c54e9052 | 268 | case 0: |
269 | httpMakePrivate(entry); | |
f5558c95 | 270 | break; |
c54e9052 | 271 | case -1: |
851eeef7 | 272 | httpCacheNegatively(entry); |
30a4f2a8 | 273 | break; |
c54e9052 | 274 | default: |
275 | assert(0); | |
4e38e700 | 276 | break; |
f5558c95 | 277 | } |
0336304c | 278 | if (reply->cache_control) { |
279 | if (EBIT_TEST(reply->cache_control->mask, CC_PROXY_REVALIDATE)) | |
0cdcddb9 | 280 | EBIT_SET(entry->flag, ENTRY_REVALIDATE); |
308e4a84 | 281 | else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE)) |
0cdcddb9 | 282 | EBIT_SET(entry->flag, ENTRY_REVALIDATE); |
0336304c | 283 | } |
9a47da71 | 284 | if (EBIT_TEST(httpState->flags, HTTP_KEEPALIVE)) |
285 | if (httpState->peer) | |
286 | httpState->peer->stats.n_keepalives_sent++; | |
9f5a2895 | 287 | if (reply->keep_alive) |
1294c0fc | 288 | if (httpState->peer) |
289 | httpState->peer->stats.n_keepalives_recv++; | |
123abbe1 | 290 | ctx_exit(ctx); |
f5558c95 | 291 | } |
292 | } | |
293 | ||
603a02fd | 294 | static int |
295 | httpPconnTransferDone(HttpStateData * httpState) | |
296 | { | |
297 | /* return 1 if we got the last of the data on a persistent connection */ | |
298 | MemObject *mem = httpState->entry->mem_obj; | |
cb69b4c7 | 299 | HttpReply *reply = mem->reply; |
51fdcbd5 | 300 | debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); |
978e455f | 301 | /* |
99edd1c3 | 302 | * If we didn't send a keep-alive request header, then this |
978e455f | 303 | * can not be a persistent connection. |
304 | */ | |
79a15e0a | 305 | if (!EBIT_TEST(httpState->flags, HTTP_KEEPALIVE)) |
603a02fd | 306 | return 0; |
9f5a2895 | 307 | /* |
308 | * What does the reply have to say about keep-alive? | |
309 | */ | |
310 | if (!reply->keep_alive) | |
311 | return 0; | |
51fdcbd5 | 312 | debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", |
d8b249ef | 313 | reply->content_length); |
603a02fd | 314 | /* |
978e455f | 315 | * Deal with gross HTTP stuff |
316 | * - If we haven't seen the end of the reply headers, we can't | |
317 | * be persistent. | |
318 | * - For "200 OK" check the content-length in the next block. | |
978e455f | 319 | * - For "204 No Content" (even with content-length) we're done. |
320 | * - For "304 Not Modified" (even with content-length) we're done. | |
a3c60429 | 321 | * - 1XX replies never have a body; we're done. |
978e455f | 322 | * - For HEAD requests with content-length we're done. |
a3c60429 | 323 | * - For all other replies, check content length in next block. |
603a02fd | 324 | */ |
978e455f | 325 | if (httpState->reply_hdr_state < 2) |
326 | return 0; | |
cb69b4c7 | 327 | else if (reply->sline.status == HTTP_OK) |
a3c60429 | 328 | (void) 0; /* common case, continue */ |
cb69b4c7 | 329 | else if (reply->sline.status == HTTP_NO_CONTENT) |
978e455f | 330 | return 1; |
cb69b4c7 | 331 | else if (reply->sline.status == HTTP_NOT_MODIFIED) |
978e455f | 332 | return 1; |
cb69b4c7 | 333 | else if (reply->sline.status < HTTP_OK) |
a3c60429 | 334 | return 1; |
978e455f | 335 | else if (httpState->request->method == METHOD_HEAD) |
336 | return 1; | |
603a02fd | 337 | /* |
a3c60429 | 338 | * If there is no content-length, then we can't be |
978e455f | 339 | * persistent. If there is a content length, then we must |
340 | * wait until we've seen the end of the body. | |
603a02fd | 341 | */ |
d8b249ef | 342 | if (reply->content_length < 0) |
603a02fd | 343 | return 0; |
d8b249ef | 344 | else if (mem->inmem_hi < reply->content_length + reply->hdr_sz) |
603a02fd | 345 | return 0; |
978e455f | 346 | else |
b34ed725 | 347 | return 1; |
603a02fd | 348 | } |
090089c4 | 349 | |
350 | /* This will be called when data is ready to be read from fd. Read until | |
351 | * error or connection closed. */ | |
f5558c95 | 352 | /* XXX this function is too long! */ |
b8d8561b | 353 | static void |
b177367b | 354 | httpReadReply(int fd, void *data) |
090089c4 | 355 | { |
b177367b | 356 | HttpStateData *httpState = data; |
95d659f0 | 357 | LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); |
bfcaf585 | 358 | StoreEntry *entry = httpState->entry; |
603a02fd | 359 | const request_t *request = httpState->request; |
090089c4 | 360 | int len; |
30a4f2a8 | 361 | int bin; |
090089c4 | 362 | int clen; |
447e176b | 363 | size_t read_sz; |
364 | #if DELAY_POOLS | |
365 | delay_id delay_id = delayMostBytesAllowed(entry->mem_obj); | |
366 | #endif | |
41462d93 | 367 | if (fwdAbortFetch(entry)) { |
9b312a19 | 368 | storeAbort(entry, 0); |
a3d5953d | 369 | comm_close(fd); |
370 | return; | |
234967c9 | 371 | } |
372 | /* check if we want to defer reading */ | |
1513873c | 373 | errno = 0; |
447e176b | 374 | read_sz = SQUID_TCP_SO_RCVBUF; |
375 | #if DELAY_POOLS | |
376 | read_sz = delayBytesWanted(delay_id, read_sz); | |
377 | assert(read_sz > 0); | |
378 | #endif | |
379 | len = read(fd, buf, read_sz); | |
a3d5953d | 380 | debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); |
30a4f2a8 | 381 | if (len > 0) { |
ee1679df | 382 | fd_bytes(fd, len, FD_READ); |
447e176b | 383 | #if DELAY_POOLS |
384 | delayBytesIn(delay_id, len); | |
385 | #endif | |
a0f32775 | 386 | kb_incr(&Counter.server.all.kbytes_in, len); |
387 | kb_incr(&Counter.server.http.kbytes_in, len); | |
4f92c80c | 388 | commSetTimeout(fd, Config.Timeout.read, NULL, NULL); |
4a63c85f | 389 | IOStats.Http.reads++; |
30a4f2a8 | 390 | for (clen = len - 1, bin = 0; clen; bin++) |
391 | clen >>= 1; | |
392 | IOStats.Http.read_hist[bin]++; | |
393 | } | |
5ede6c8f | 394 | if (!httpState->reply_hdr && len > 0) { |
395 | /* Skip whitespace */ | |
396 | while (len > 0 && isspace(*buf)) | |
397 | xmemmove(buf, buf + 1, len--); | |
398 | if (len == 0) { | |
399 | /* Continue to read... */ | |
400 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
401 | return; | |
402 | } | |
403 | } | |
ba718c8f | 404 | if (len < 0) { |
55cb44f1 | 405 | debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n", |
406 | fd, xstrerror()); | |
b224ea98 | 407 | if (ignoreErrno(errno)) { |
9b312a19 | 408 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); |
910169e5 | 409 | } else if (entry->mem_obj->inmem_hi == 0) { |
410 | fwdFail(httpState->fwdState, ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR, errno); | |
1afe05c5 | 411 | comm_close(fd); |
090089c4 | 412 | } else { |
55cb44f1 | 413 | storeAbort(entry, 0); |
0d4d4170 | 414 | comm_close(fd); |
090089c4 | 415 | } |
8350fe9b | 416 | } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { |
910169e5 | 417 | fwdFail(httpState->fwdState, ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE, errno); |
418 | httpState->eof = 1; | |
419 | comm_close(fd); | |
090089c4 | 420 | } else if (len == 0) { |
421 | /* Connection closed; retrieval done. */ | |
f86a6a46 | 422 | httpState->eof = 1; |
d1a43e28 | 423 | if (httpState->reply_hdr_state < 2) |
b34ed725 | 424 | /* |
425 | * Yes Henrik, there is a point to doing this. When we | |
426 | * called httpProcessReplyHeader() before, we didn't find | |
427 | * the end of headers, but now we are definately at EOF, so | |
428 | * we want to process the reply headers. | |
429 | */ | |
d1a43e28 | 430 | httpProcessReplyHeader(httpState, buf, len); |
d1a43e28 | 431 | storeComplete(entry); /* deallocates mem_obj->request */ |
0d4d4170 | 432 | comm_close(fd); |
090089c4 | 433 | } else { |
d1a43e28 | 434 | if (httpState->reply_hdr_state < 2) |
30a4f2a8 | 435 | httpProcessReplyHeader(httpState, buf, len); |
620da955 | 436 | storeAppend(entry, buf, len); |
603a02fd | 437 | if (httpPconnTransferDone(httpState)) { |
5b29969a | 438 | /* yes we have to clear all these! */ |
8796b9e9 | 439 | commSetDefer(fd, NULL, NULL); |
5b29969a | 440 | commSetTimeout(fd, -1, NULL, NULL); |
441 | commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); | |
603a02fd | 442 | comm_remove_close_handler(fd, httpStateFree, httpState); |
443 | storeComplete(entry); /* deallocates mem_obj->request */ | |
52f0d243 | 444 | /* call storeComplete BEFORE fwdUnregister or else fwdUnregister |
445 | * will storeAbort */ | |
446 | fwdUnregister(fd, httpState->fwdState); | |
8796b9e9 | 447 | pconnPush(fd, request->host, request->port); |
603a02fd | 448 | httpState->fd = -1; |
449 | httpStateFree(-1, httpState); | |
450 | } else { | |
9f5a2895 | 451 | /* Wait for EOF condition */ |
603a02fd | 452 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); |
453 | } | |
090089c4 | 454 | } |
455 | } | |
456 | ||
457 | /* This will be called when request write is complete. Schedule read of | |
458 | * reply. */ | |
b8d8561b | 459 | static void |
79a15e0a | 460 | httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) |
090089c4 | 461 | { |
30a4f2a8 | 462 | HttpStateData *httpState = data; |
9b312a19 | 463 | StoreEntry *entry = httpState->entry; |
464 | ErrorState *err; | |
a3d5953d | 465 | debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n", |
090089c4 | 466 | fd, size, errflag); |
ee1679df | 467 | if (size > 0) { |
468 | fd_bytes(fd, size, FD_WRITE); | |
a0f32775 | 469 | kb_incr(&Counter.server.all.kbytes_out, size); |
399e85ea | 470 | kb_incr(&Counter.server.http.kbytes_out, size); |
ee1679df | 471 | } |
ea3a2a69 | 472 | if (errflag == COMM_ERR_CLOSING) |
473 | return; | |
090089c4 | 474 | if (errflag) { |
fe40a877 | 475 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); |
c45ed9ad | 476 | err->xerrno = errno; |
79a15e0a | 477 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 478 | errorAppendEntry(entry, err); |
0d4d4170 | 479 | comm_close(fd); |
090089c4 | 480 | return; |
481 | } else { | |
482 | /* Schedule read reply. */ | |
b177367b | 483 | commSetSelect(fd, |
019dd986 | 484 | COMM_SELECT_READ, |
b177367b | 485 | httpReadReply, |
cd1fb0eb | 486 | httpState, 0); |
41462d93 | 487 | commSetDefer(fd, fwdCheckDeferRead, entry); |
090089c4 | 488 | } |
489 | } | |
490 | ||
99edd1c3 | 491 | /* |
492 | * build request headers and append them to a given MemBuf | |
493 | * used by httpBuildRequestPrefix() | |
494 | * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing | |
495 | */ | |
e1e72f06 | 496 | void |
6bf8443a | 497 | httpBuildRequestHeader(request_t * request, |
498 | request_t * orig_request, | |
499 | StoreEntry * entry, | |
5999b776 | 500 | HttpHeader * hdr_out, |
603a02fd | 501 | int cfd, |
502 | int flags) | |
6bf8443a | 503 | { |
99edd1c3 | 504 | /* building buffer for complex strings */ |
5999b776 | 505 | #define BBUF_SZ (MAX_URL+32) |
99edd1c3 | 506 | LOCAL_ARRAY(char, bbuf, BBUF_SZ); |
507 | String strConnection = StringNull; | |
508 | const HttpHeader *hdr_in = &orig_request->header; | |
d192d11f | 509 | int filter_range; |
99edd1c3 | 510 | const HttpHeaderEntry *e; |
511 | HttpHeaderPos pos = HttpHeaderInitPos; | |
2246b732 | 512 | httpHeaderInit(hdr_out, hoRequest); |
99edd1c3 | 513 | /* append our IMS header */ |
e17dc75c | 514 | if (entry && entry->lastmod > -1 && request->method == METHOD_GET) |
99edd1c3 | 515 | httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, entry->lastmod); |
516 | ||
137ee196 | 517 | /* decide if we want to filter out Range specs |
518 | * no reason to filter out if the reply will not be cachable | |
519 | * or if we cannot parse the specs */ | |
d192d11f | 520 | filter_range = |
521 | orig_request->range && EBIT_TEST(orig_request->flags, REQ_CACHABLE); | |
137ee196 | 522 | |
99edd1c3 | 523 | strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); |
524 | while ((e = httpHeaderGetEntry(hdr_in, &pos))) { | |
525 | debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", | |
526 | strBuf(e->name), strBuf(e->value)); | |
527 | if (!httpRequestHdrAllowed(e, &strConnection)) | |
6bf8443a | 528 | continue; |
99edd1c3 | 529 | switch (e->id) { |
530 | case HDR_PROXY_AUTHORIZATION: | |
afe95a7e | 531 | /* If we're not going to do proxy auth, then it must be passed on */ |
99edd1c3 | 532 | if (!EBIT_TEST(request->flags, REQ_USED_PROXY_AUTH)) |
533 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
534 | break; | |
535 | case HDR_HOST: | |
77ed547a | 536 | /* Don't use client's Host: header for redirected requests */ |
99edd1c3 | 537 | if (!EBIT_TEST(request->flags, REQ_REDIRECTED)) |
538 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
539 | break; | |
540 | case HDR_IF_MODIFIED_SINCE: | |
541 | /* append unless we added our own; | |
542 | * note: at most one client's ims header can pass through */ | |
543 | if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) | |
544 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
545 | break; | |
546 | case HDR_MAX_FORWARDS: | |
b3b64e58 | 547 | if (orig_request->method == METHOD_TRACE) { |
99edd1c3 | 548 | /* sacrificing efficiency over clarity, etc. */ |
549 | const int hops = httpHeaderGetInt(hdr_in, HDR_MAX_FORWARDS); | |
550 | if (hops > 0) | |
5999b776 | 551 | httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); |
b3b64e58 | 552 | } |
99edd1c3 | 553 | break; |
137ee196 | 554 | case HDR_RANGE: |
a9771e51 | 555 | case HDR_IF_RANGE: |
d192d11f | 556 | if (!filter_range) |
137ee196 | 557 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); |
558 | break; | |
99edd1c3 | 559 | case HDR_PROXY_CONNECTION: |
560 | case HDR_CONNECTION: | |
561 | case HDR_VIA: | |
562 | case HDR_X_FORWARDED_FOR: | |
563 | case HDR_CACHE_CONTROL: | |
564 | /* append these after the loop if needed */ | |
565 | break; | |
566 | default: | |
567 | /* pass on all other header fields */ | |
568 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
66f7337b | 569 | } |
88738790 | 570 | } |
99edd1c3 | 571 | |
572 | /* append fake user agent if configured and | |
573 | * the real one is not supplied by the client */ | |
574 | if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT)) | |
575 | httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua); | |
576 | ||
577 | /* append Via */ | |
578 | { | |
579 | String strVia = httpHeaderGetList(hdr_in, HDR_VIA); | |
580 | snprintf(bbuf, BBUF_SZ, "%3.1f %s", orig_request->http_ver, ThisCache); | |
581 | strListAdd(&strVia, bbuf, ','); | |
582 | httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); | |
583 | stringClean(&strVia); | |
6bf8443a | 584 | } |
99edd1c3 | 585 | /* append X-Forwarded-For */ |
586 | { | |
587 | String strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); | |
588 | strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); | |
589 | httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); | |
590 | stringClean(&strFwd); | |
591 | } | |
592 | /* append Host if not there already */ | |
593 | if (!httpHeaderHas(hdr_out, HDR_HOST)) { | |
594 | /* use port# only if not default */ | |
595 | if (orig_request->port == urlDefaultPort(orig_request->protocol)) { | |
596 | httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host); | |
597 | } else { | |
2246b732 | 598 | httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d", |
99edd1c3 | 599 | orig_request->host, (int) orig_request->port); |
99edd1c3 | 600 | } |
6bf8443a | 601 | } |
99edd1c3 | 602 | /* append Cache-Control, add max-age if not there already */ |
603 | { | |
604 | HttpHdrCc *cc = httpHeaderGetCc(hdr_in); | |
605 | if (!cc) | |
606 | cc = httpHdrCcCreate(); | |
607 | if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) { | |
9b5d1d21 | 608 | const char *url = entry ? storeUrl(entry) : urlCanonical(orig_request); |
99edd1c3 | 609 | httpHdrCcSetMaxAge(cc, getMaxAge(url)); |
610 | if (strLen(request->urlpath)) | |
611 | assert(strstr(url, strBuf(request->urlpath))); | |
612 | } | |
613 | httpHeaderPutCc(hdr_out, cc); | |
614 | httpHdrCcDestroy(cc); | |
6bf8443a | 615 | } |
99edd1c3 | 616 | /* maybe append Connection: keep-alive */ |
79a15e0a | 617 | if (EBIT_TEST(flags, HTTP_KEEPALIVE)) { |
618 | if (EBIT_TEST(flags, HTTP_PROXYING)) { | |
99edd1c3 | 619 | httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive"); |
603a02fd | 620 | } else { |
99edd1c3 | 621 | httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive"); |
603a02fd | 622 | } |
603a02fd | 623 | } |
99edd1c3 | 624 | stringClean(&strConnection); |
625 | } | |
626 | ||
627 | /* build request prefix and append it to a given MemBuf; | |
628 | * return the length of the prefix */ | |
629 | size_t | |
630 | httpBuildRequestPrefix(request_t * request, | |
631 | request_t * orig_request, | |
632 | StoreEntry * entry, | |
5999b776 | 633 | MemBuf * mb, |
99edd1c3 | 634 | int cfd, |
635 | int flags) | |
636 | { | |
637 | const int offset = mb->size; | |
638 | memBufPrintf(mb, "%s %s HTTP/1.0\r\n", | |
639 | RequestMethodStr[request->method], | |
640 | strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); | |
641 | /* build and pack headers */ | |
642 | { | |
643 | HttpHeader hdr; | |
644 | Packer p; | |
645 | httpBuildRequestHeader(request, orig_request, entry, &hdr, cfd, flags); | |
646 | packerToMemInit(&p, mb); | |
647 | httpHeaderPackInto(&hdr, &p); | |
648 | httpHeaderClean(&hdr); | |
649 | packerClean(&p); | |
9d9d144b | 650 | } |
99edd1c3 | 651 | /* append header terminator */ |
652 | memBufAppend(mb, "\r\n", 2); | |
653 | return mb->size - offset; | |
6bf8443a | 654 | } |
655 | ||
090089c4 | 656 | /* This will be called when connect completes. Write request. */ |
b8d8561b | 657 | static void |
b177367b | 658 | httpSendRequest(int fd, void *data) |
090089c4 | 659 | { |
b177367b | 660 | HttpStateData *httpState = data; |
99edd1c3 | 661 | MemBuf mb; |
30a4f2a8 | 662 | request_t *req = httpState->request; |
620da955 | 663 | StoreEntry *entry = httpState->entry; |
2a26c096 | 664 | int cfd; |
1294c0fc | 665 | peer *p = httpState->peer; |
901e234d | 666 | CWCB *sendHeaderDone; |
090089c4 | 667 | |
a3d5953d | 668 | debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", fd, httpState); |
090089c4 | 669 | |
efb9218c | 670 | if (pumpMethod(req->method)) |
7db8b16d | 671 | sendHeaderDone = httpSendRequestEntry; |
672 | else | |
673 | sendHeaderDone = httpSendComplete; | |
54220df8 | 674 | |
2a26c096 | 675 | if (!opt_forwarded_for) |
6bf8443a | 676 | cfd = -1; |
2a26c096 | 677 | else if (entry->mem_obj == NULL) |
6bf8443a | 678 | cfd = -1; |
2a26c096 | 679 | else |
382d851a | 680 | cfd = entry->mem_obj->fd; |
b0a1e5bf | 681 | assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type); |
1294c0fc | 682 | if (p != NULL) |
79a15e0a | 683 | EBIT_SET(httpState->flags, HTTP_PROXYING); |
efb9218c | 684 | /* |
99edd1c3 | 685 | * Is keep-alive okay for all request methods? |
efb9218c | 686 | */ |
687 | if (p == NULL) | |
688 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); | |
689 | else if (p->stats.n_keepalives_sent < 10) | |
690 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); | |
691 | else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) | |
692 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); | |
99edd1c3 | 693 | memBufDefInit(&mb); |
694 | httpBuildRequestPrefix(req, | |
79a15e0a | 695 | httpState->orig_request, |
6bf8443a | 696 | entry, |
99edd1c3 | 697 | &mb, |
603a02fd | 698 | cfd, |
699 | httpState->flags); | |
99edd1c3 | 700 | debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", fd, mb.buf); |
701 | comm_write_mbuf(fd, mb, sendHeaderDone, httpState); | |
090089c4 | 702 | } |
703 | ||
910169e5 | 704 | void |
705 | httpStart(FwdState * fwdState, int fd) | |
603a02fd | 706 | { |
cb87dab6 | 707 | HttpStateData *httpState = memAllocate(MEM_HTTP_STATE_DATA); |
910169e5 | 708 | request_t *proxy_req; |
709 | request_t *orig_req = fwdState->request; | |
710 | debug(11, 3) ("httpStart: \"%s %s\"\n", | |
711 | RequestMethodStr[orig_req->method], | |
712 | storeUrl(fwdState->entry)); | |
cb87dab6 | 713 | cbdataAdd(httpState, MEM_HTTP_STATE_DATA); |
910169e5 | 714 | storeLockObject(fwdState->entry); |
715 | httpState->fwdState = fwdState; | |
716 | httpState->entry = fwdState->entry; | |
9e4ad609 | 717 | httpState->fd = fd; |
910169e5 | 718 | if (fwdState->servers) |
719 | httpState->peer = fwdState->servers->peer; /* might be NULL */ | |
720 | if (httpState->peer) { | |
721 | proxy_req = requestCreate(orig_req->method, | |
722 | PROTO_NONE, storeUrl(httpState->entry)); | |
723 | xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN); | |
724 | proxy_req->port = httpState->peer->http_port; | |
23e8446b | 725 | proxy_req->flags = orig_req->flags; |
910169e5 | 726 | httpState->request = requestLink(proxy_req); |
910169e5 | 727 | httpState->orig_request = requestLink(orig_req); |
728 | EBIT_SET(proxy_req->flags, REQ_PROXYING); | |
729 | /* | |
730 | * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here. | |
731 | * We might end up getting the object from somewhere else if, | |
732 | * for example, the request to this neighbor fails. | |
733 | */ | |
734 | if (EBIT_TEST(httpState->peer->options, NEIGHBOR_PROXY_ONLY)) | |
735 | storeReleaseRequest(httpState->entry); | |
95e36d02 | 736 | #if DELAY_POOLS |
737 | if (EBIT_TEST(httpState->peer->options, NEIGHBOR_NO_DELAY)) { | |
447e176b | 738 | proxy_req->delay_id = 0; |
95e36d02 | 739 | } else { |
447e176b | 740 | proxy_req->delay_id = orig_req->delay_id; |
95e36d02 | 741 | } |
742 | #endif | |
603a02fd | 743 | } else { |
910169e5 | 744 | httpState->request = requestLink(orig_req); |
745 | httpState->orig_request = requestLink(orig_req); | |
603a02fd | 746 | } |
910169e5 | 747 | /* |
748 | * register the handler to free HTTP state data when the FD closes | |
749 | */ | |
750 | comm_add_close_handler(fd, httpStateFree, httpState); | |
a0f32775 | 751 | Counter.server.all.requests++; |
752 | Counter.server.http.requests++; | |
41462d93 | 753 | httpConnectDone(fd, COMM_OK, httpState); |
e5f6c5c2 | 754 | } |
755 | ||
756 | static void | |
757 | httpConnectDone(int fd, int status, void *data) | |
758 | { | |
759 | HttpStateData *httpState = data; | |
760 | request_t *request = httpState->request; | |
761 | StoreEntry *entry = httpState->entry; | |
9b312a19 | 762 | ErrorState *err; |
edeb28fd | 763 | if (status == COMM_ERR_DNS) { |
a3d5953d | 764 | debug(11, 4) ("httpConnectDone: Unknown host: %s\n", request->host); |
fe40a877 | 765 | err = errorCon(ERR_DNS_FAIL, HTTP_SERVICE_UNAVAILABLE); |
9b312a19 | 766 | err->dnsserver_msg = xstrdup(dns_error_message); |
79a15e0a | 767 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 768 | errorAppendEntry(entry, err); |
edeb28fd | 769 | comm_close(fd); |
770 | } else if (status != COMM_OK) { | |
fe40a877 | 771 | err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE); |
c45ed9ad | 772 | err->xerrno = errno; |
9b312a19 | 773 | err->host = xstrdup(request->host); |
774 | err->port = request->port; | |
79a15e0a | 775 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 776 | errorAppendEntry(entry, err); |
1294c0fc | 777 | if (httpState->peer) |
778 | peerCheckConnectStart(httpState->peer); | |
e5f6c5c2 | 779 | comm_close(fd); |
780 | } else { | |
bfcaf585 | 781 | commSetSelect(fd, COMM_SELECT_WRITE, httpSendRequest, httpState, 0); |
86cf9987 | 782 | commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); |
090089c4 | 783 | } |
090089c4 | 784 | } |
785 | ||
54220df8 | 786 | static void |
7db8b16d | 787 | httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void *data) |
54220df8 | 788 | { |
789 | HttpStateData *httpState = data; | |
790 | StoreEntry *entry = httpState->entry; | |
791 | ErrorState *err; | |
792 | debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n", | |
7db8b16d | 793 | fd, size, errflag); |
54220df8 | 794 | if (size > 0) { |
7db8b16d | 795 | fd_bytes(fd, size, FD_WRITE); |
54220df8 | 796 | kb_incr(&Counter.server.all.kbytes_out, size); |
797 | kb_incr(&Counter.server.http.kbytes_out, size); | |
798 | } | |
799 | if (errflag == COMM_ERR_CLOSING) | |
7db8b16d | 800 | return; |
54220df8 | 801 | if (errflag) { |
7db8b16d | 802 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); |
803 | err->xerrno = errno; | |
804 | err->request = requestLink(httpState->orig_request); | |
805 | errorAppendEntry(entry, err); | |
806 | comm_close(fd); | |
807 | return; | |
54220df8 | 808 | } |
7db8b16d | 809 | pumpStart(fd, entry, httpState->orig_request, httpSendComplete, httpState); |
54220df8 | 810 | } |