]>
Commit | Line | Data |
---|---|---|
1 | ||
2 | /* | |
3 | * $Id: http.cc,v 1.354 1999/10/04 05:05:15 wessels Exp $ | |
4 | * | |
5 | * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) | |
6 | * AUTHOR: Harvest Derived | |
7 | * | |
8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ | |
9 | * ---------------------------------------------------------- | |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
13 | * National Laboratory for Applied Network Research and funded by the | |
14 | * National Science Foundation. Squid is Copyrighted (C) 1998 by | |
15 | * Duane Wessels and the University of California San Diego. Please | |
16 | * see the COPYRIGHT file for full details. Squid incorporates | |
17 | * software developed and/or copyrighted by other sources. Please see | |
18 | * the CREDITS file for full details. | |
19 | * | |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
36 | /* | |
37 | * Anonymizing patch by lutz@as-node.jena.thur.de | |
38 | * have a look into http-anon.c to get more informations. | |
39 | */ | |
40 | ||
41 | #include "squid.h" | |
42 | ||
43 | static const char *const crlf = "\r\n"; | |
44 | ||
45 | static CWCB httpSendComplete; | |
46 | static CWCB httpSendRequestEntry; | |
47 | static CWCB httpSendRequestEntryDone; | |
48 | ||
49 | static PF httpReadReply; | |
50 | static void httpSendRequest(HttpStateData *); | |
51 | static PF httpStateFree; | |
52 | static PF httpTimeout; | |
53 | static void httpCacheNegatively(StoreEntry *); | |
54 | static void httpMakePrivate(StoreEntry *); | |
55 | static void httpMakePublic(StoreEntry *); | |
56 | static int httpCachableReply(HttpStateData *); | |
57 | static void httpMaybeRemovePublic(StoreEntry *, http_status); | |
58 | ||
59 | static void | |
60 | httpStateFree(int fd, void *data) | |
61 | { | |
62 | HttpStateData *httpState = data; | |
63 | #if DELAY_POOLS | |
64 | delayClearNoDelay(fd); | |
65 | #endif | |
66 | if (httpState == NULL) | |
67 | return; | |
68 | storeUnlockObject(httpState->entry); | |
69 | if (httpState->reply_hdr) { | |
70 | memFree(httpState->reply_hdr, MEM_8K_BUF); | |
71 | httpState->reply_hdr = NULL; | |
72 | } | |
73 | requestUnlink(httpState->request); | |
74 | requestUnlink(httpState->orig_request); | |
75 | httpState->request = NULL; | |
76 | httpState->orig_request = NULL; | |
77 | cbdataFree(httpState); | |
78 | } | |
79 | ||
80 | int | |
81 | httpCachable(method_t method) | |
82 | { | |
83 | /* GET and HEAD are cachable. Others are not. */ | |
84 | if (method != METHOD_GET && method != METHOD_HEAD) | |
85 | return 0; | |
86 | /* else cachable */ | |
87 | return 1; | |
88 | } | |
89 | ||
90 | static void | |
91 | httpTimeout(int fd, void *data) | |
92 | { | |
93 | HttpStateData *httpState = data; | |
94 | StoreEntry *entry = httpState->entry; | |
95 | debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); | |
96 | if (entry->store_status == STORE_PENDING) { | |
97 | if (entry->mem_obj->inmem_hi == 0) { | |
98 | fwdFail(httpState->fwd, | |
99 | errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); | |
100 | } | |
101 | } | |
102 | comm_close(fd); | |
103 | } | |
104 | ||
105 | /* This object can be cached for a long time */ | |
106 | static void | |
107 | httpMakePublic(StoreEntry * entry) | |
108 | { | |
109 | if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) | |
110 | storeSetPublicKey(entry); | |
111 | } | |
112 | ||
113 | /* This object should never be cached at all */ | |
114 | static void | |
115 | httpMakePrivate(StoreEntry * entry) | |
116 | { | |
117 | storeExpireNow(entry); | |
118 | storeReleaseRequest(entry); /* delete object when not used */ | |
119 | /* storeReleaseRequest clears ENTRY_CACHABLE flag */ | |
120 | } | |
121 | ||
122 | /* This object may be negatively cached */ | |
123 | static void | |
124 | httpCacheNegatively(StoreEntry * entry) | |
125 | { | |
126 | storeNegativeCache(entry); | |
127 | if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) | |
128 | storeSetPublicKey(entry); | |
129 | } | |
130 | ||
131 | static void | |
132 | httpMaybeRemovePublic(StoreEntry * e, http_status status) | |
133 | { | |
134 | int remove = 0; | |
135 | StoreEntry *pe; | |
136 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) | |
137 | return; | |
138 | switch (status) { | |
139 | case HTTP_OK: | |
140 | case HTTP_NON_AUTHORITATIVE_INFORMATION: | |
141 | case HTTP_MULTIPLE_CHOICES: | |
142 | case HTTP_MOVED_PERMANENTLY: | |
143 | case HTTP_MOVED_TEMPORARILY: | |
144 | case HTTP_FORBIDDEN: | |
145 | case HTTP_NOT_FOUND: | |
146 | case HTTP_METHOD_NOT_ALLOWED: | |
147 | case HTTP_GONE: | |
148 | remove = 1; | |
149 | break; | |
150 | #if WORK_IN_PROGRESS | |
151 | case HTTP_UNAUTHORIZED: | |
152 | remove = 1; | |
153 | break; | |
154 | #endif | |
155 | default: | |
156 | remove = 0; | |
157 | break; | |
158 | } | |
159 | if (!remove) | |
160 | return; | |
161 | assert(e->mem_obj); | |
162 | if ((pe = storeGetPublic(e->mem_obj->url, e->mem_obj->method)) != NULL) { | |
163 | assert(e != pe); | |
164 | storeRelease(pe); | |
165 | } | |
166 | if (e->mem_obj->method == METHOD_GET) { | |
167 | /* A fresh GET should eject old HEAD objects */ | |
168 | if ((pe = storeGetPublic(e->mem_obj->url, METHOD_HEAD)) != NULL) { | |
169 | assert(e != pe); | |
170 | storeRelease(pe); | |
171 | } | |
172 | } | |
173 | } | |
174 | ||
175 | static int | |
176 | httpCachableReply(HttpStateData * httpState) | |
177 | { | |
178 | HttpReply *rep = httpState->entry->mem_obj->reply; | |
179 | HttpHeader *hdr = &rep->header; | |
180 | const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0; | |
181 | const char *v; | |
182 | if (EBIT_TEST(cc_mask, CC_PRIVATE)) | |
183 | return 0; | |
184 | if (EBIT_TEST(cc_mask, CC_NO_CACHE)) | |
185 | return 0; | |
186 | if (EBIT_TEST(cc_mask, CC_NO_STORE)) | |
187 | return 0; | |
188 | if (httpState->request->flags.auth) { | |
189 | /* | |
190 | * Responses to requests with authorization may be cached | |
191 | * only if a Cache-Control: public reply header is present. | |
192 | * RFC 2068, sec 14.9.4 | |
193 | */ | |
194 | if (!EBIT_TEST(cc_mask, CC_PUBLIC)) | |
195 | return 0; | |
196 | } | |
197 | /* | |
198 | * We don't properly deal with Vary features yet, so we can't | |
199 | * cache these | |
200 | */ | |
201 | if (httpHeaderHas(hdr, HDR_VARY)) | |
202 | return 0; | |
203 | /* Pragma: no-cache in _replies_ is not documented in HTTP, | |
204 | * but servers like "Active Imaging Webcast/2.0" sure do use it */ | |
205 | if (httpHeaderHas(hdr, HDR_PRAGMA)) { | |
206 | String s = httpHeaderGetList(hdr, HDR_PRAGMA); | |
207 | const int no_cache = strListIsMember(&s, "no-cache", ','); | |
208 | stringClean(&s); | |
209 | if (no_cache) | |
210 | return 0; | |
211 | } | |
212 | /* | |
213 | * The "multipart/x-mixed-replace" content type is used for | |
214 | * continuous push replies. These are generally dynamic and | |
215 | * probably should not be cachable | |
216 | */ | |
217 | if ((v = httpHeaderGetStr(hdr, HDR_CONTENT_TYPE))) | |
218 | if (!strncasecmp(v, "multipart/x-mixed-replace", 25)) | |
219 | return 0; | |
220 | switch (httpState->entry->mem_obj->reply->sline.status) { | |
221 | /* Responses that are cacheable */ | |
222 | case HTTP_OK: | |
223 | case HTTP_NON_AUTHORITATIVE_INFORMATION: | |
224 | case HTTP_MULTIPLE_CHOICES: | |
225 | case HTTP_MOVED_PERMANENTLY: | |
226 | case HTTP_GONE: | |
227 | /* | |
228 | * Don't cache objects that need to be refreshed on next request, | |
229 | * unless we know how to refresh it. | |
230 | */ | |
231 | if (!refreshIsCachable(httpState->entry)) | |
232 | return 0; | |
233 | /* don't cache objects from peers w/o LMT, Date, or Expires */ | |
234 | /* check that is it enough to check headers @?@ */ | |
235 | if (rep->date > -1) | |
236 | return 1; | |
237 | else if (rep->last_modified > -1) | |
238 | return 1; | |
239 | else if (!httpState->peer) | |
240 | return 1; | |
241 | /* @?@ (here and 302): invalid expires header compiles to squid_curtime */ | |
242 | else if (rep->expires > -1) | |
243 | return 1; | |
244 | else | |
245 | return 0; | |
246 | /* NOTREACHED */ | |
247 | break; | |
248 | /* Responses that only are cacheable if the server says so */ | |
249 | case HTTP_MOVED_TEMPORARILY: | |
250 | if (rep->expires > -1) | |
251 | return 1; | |
252 | else | |
253 | return 0; | |
254 | /* NOTREACHED */ | |
255 | break; | |
256 | /* Errors can be negatively cached */ | |
257 | case HTTP_NO_CONTENT: | |
258 | case HTTP_USE_PROXY: | |
259 | case HTTP_BAD_REQUEST: | |
260 | case HTTP_FORBIDDEN: | |
261 | case HTTP_NOT_FOUND: | |
262 | case HTTP_METHOD_NOT_ALLOWED: | |
263 | case HTTP_REQUEST_URI_TOO_LARGE: | |
264 | case HTTP_INTERNAL_SERVER_ERROR: | |
265 | case HTTP_NOT_IMPLEMENTED: | |
266 | case HTTP_BAD_GATEWAY: | |
267 | case HTTP_SERVICE_UNAVAILABLE: | |
268 | case HTTP_GATEWAY_TIMEOUT: | |
269 | return -1; | |
270 | /* NOTREACHED */ | |
271 | break; | |
272 | /* Some responses can never be cached */ | |
273 | case HTTP_PARTIAL_CONTENT: /* Not yet supported */ | |
274 | case HTTP_SEE_OTHER: | |
275 | case HTTP_NOT_MODIFIED: | |
276 | case HTTP_UNAUTHORIZED: | |
277 | case HTTP_PROXY_AUTHENTICATION_REQUIRED: | |
278 | case HTTP_INVALID_HEADER: /* Squid header parsing error */ | |
279 | default: /* Unknown status code */ | |
280 | return 0; | |
281 | /* NOTREACHED */ | |
282 | break; | |
283 | } | |
284 | /* NOTREACHED */ | |
285 | } | |
286 | ||
287 | /* rewrite this later using new interfaces @?@ */ | |
288 | void | |
289 | httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) | |
290 | { | |
291 | char *t = NULL; | |
292 | StoreEntry *entry = httpState->entry; | |
293 | int room; | |
294 | size_t hdr_len; | |
295 | HttpReply *reply = entry->mem_obj->reply; | |
296 | Ctx ctx; | |
297 | debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", | |
298 | storeKeyText(entry->key)); | |
299 | if (httpState->reply_hdr == NULL) | |
300 | httpState->reply_hdr = memAllocate(MEM_8K_BUF); | |
301 | assert(httpState->reply_hdr_state == 0); | |
302 | hdr_len = strlen(httpState->reply_hdr); | |
303 | room = 8191 - hdr_len; | |
304 | strncat(httpState->reply_hdr, buf, room < size ? room : size); | |
305 | hdr_len += room < size ? room : size; | |
306 | if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { | |
307 | debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); | |
308 | httpState->reply_hdr_state += 2; | |
309 | reply->sline.status = HTTP_INVALID_HEADER; | |
310 | return; | |
311 | } | |
312 | t = httpState->reply_hdr + hdr_len; | |
313 | /* headers can be incomplete only if object still arriving */ | |
314 | if (!httpState->eof) { | |
315 | size_t k = headersEnd(httpState->reply_hdr, 8192); | |
316 | if (0 == k) | |
317 | return; /* headers not complete */ | |
318 | t = httpState->reply_hdr + k; | |
319 | } | |
320 | *t = '\0'; | |
321 | httpState->reply_hdr_state++; | |
322 | assert(httpState->reply_hdr_state == 1); | |
323 | ctx = ctx_enter(entry->mem_obj->url); | |
324 | httpState->reply_hdr_state++; | |
325 | debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", | |
326 | httpState->reply_hdr); | |
327 | /* Parse headers into reply structure */ | |
328 | /* what happens if we fail to parse here? */ | |
329 | httpReplyParse(reply, httpState->reply_hdr, hdr_len); | |
330 | storeTimestampsSet(entry); | |
331 | /* Check if object is cacheable or not based on reply code */ | |
332 | debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); | |
333 | if (neighbors_do_private_keys) | |
334 | httpMaybeRemovePublic(entry, reply->sline.status); | |
335 | switch (httpCachableReply(httpState)) { | |
336 | case 1: | |
337 | httpMakePublic(entry); | |
338 | break; | |
339 | case 0: | |
340 | httpMakePrivate(entry); | |
341 | break; | |
342 | case -1: | |
343 | httpCacheNegatively(entry); | |
344 | break; | |
345 | default: | |
346 | assert(0); | |
347 | break; | |
348 | } | |
349 | if (reply->cache_control) { | |
350 | if (EBIT_TEST(reply->cache_control->mask, CC_PROXY_REVALIDATE)) | |
351 | EBIT_SET(entry->flags, ENTRY_REVALIDATE); | |
352 | else if (EBIT_TEST(reply->cache_control->mask, CC_MUST_REVALIDATE)) | |
353 | EBIT_SET(entry->flags, ENTRY_REVALIDATE); | |
354 | } | |
355 | if (httpState->flags.keepalive) | |
356 | if (httpState->peer) | |
357 | httpState->peer->stats.n_keepalives_sent++; | |
358 | if (reply->keep_alive) | |
359 | if (httpState->peer) | |
360 | httpState->peer->stats.n_keepalives_recv++; | |
361 | if (reply->date > -1 && !httpState->peer) { | |
362 | int skew = abs(reply->date - squid_curtime); | |
363 | if (skew > 86400) | |
364 | debug(11, 3) ("%s's clock is skewed by %d seconds!\n", | |
365 | httpState->request->host, skew); | |
366 | } | |
367 | ctx_exit(ctx); | |
368 | } | |
369 | ||
370 | static int | |
371 | httpPconnTransferDone(HttpStateData * httpState) | |
372 | { | |
373 | /* return 1 if we got the last of the data on a persistent connection */ | |
374 | MemObject *mem = httpState->entry->mem_obj; | |
375 | HttpReply *reply = mem->reply; | |
376 | int clen; | |
377 | debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); | |
378 | /* | |
379 | * If we didn't send a keep-alive request header, then this | |
380 | * can not be a persistent connection. | |
381 | */ | |
382 | if (!httpState->flags.keepalive) | |
383 | return 0; | |
384 | /* | |
385 | * What does the reply have to say about keep-alive? | |
386 | */ | |
387 | /* | |
388 | * XXX BUG? | |
389 | * If the origin server (HTTP/1.0) does not send a keep-alive | |
390 | * header, but keeps the connection open anyway, what happens? | |
391 | * We'll return here and http.c waits for an EOF before changing | |
392 | * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT | |
393 | * and an error status code, and we might have to wait until | |
394 | * the server times out the socket. | |
395 | */ | |
396 | if (!reply->keep_alive) | |
397 | return 0; | |
398 | debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", | |
399 | reply->content_length); | |
400 | /* If we haven't seen the end of reply headers, we are not done */ | |
401 | if (httpState->reply_hdr_state < 2) | |
402 | return 0; | |
403 | clen = httpReplyBodySize(httpState->request->method, reply); | |
404 | /* If there is no message body, we can be persistent */ | |
405 | if (0 == clen) | |
406 | return 1; | |
407 | /* If the body size is unknown we must wait for EOF */ | |
408 | if (clen < 0) | |
409 | return 0; | |
410 | /* If the body size is known, we must wait until we've gotten all of it. */ | |
411 | if (mem->inmem_hi < reply->content_length + reply->hdr_sz) | |
412 | return 0; | |
413 | /* We got it all */ | |
414 | return 1; | |
415 | } | |
416 | ||
417 | /* This will be called when data is ready to be read from fd. Read until | |
418 | * error or connection closed. */ | |
419 | /* XXX this function is too long! */ | |
420 | static void | |
421 | httpReadReply(int fd, void *data) | |
422 | { | |
423 | HttpStateData *httpState = data; | |
424 | LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); | |
425 | StoreEntry *entry = httpState->entry; | |
426 | const request_t *request = httpState->request; | |
427 | int len; | |
428 | int bin; | |
429 | int clen; | |
430 | size_t read_sz; | |
431 | #if DELAY_POOLS | |
432 | delay_id delay_id; | |
433 | ||
434 | /* special "if" only for http (for nodelay proxy conns) */ | |
435 | if (delayIsNoDelay(fd)) | |
436 | delay_id = 0; | |
437 | else | |
438 | delay_id = delayMostBytesAllowed(entry->mem_obj); | |
439 | #endif | |
440 | if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { | |
441 | comm_close(fd); | |
442 | return; | |
443 | } | |
444 | /* check if we want to defer reading */ | |
445 | errno = 0; | |
446 | read_sz = SQUID_TCP_SO_RCVBUF; | |
447 | #if DELAY_POOLS | |
448 | read_sz = delayBytesWanted(delay_id, 1, read_sz); | |
449 | #endif | |
450 | Counter.syscalls.sock.reads++; | |
451 | len = read(fd, buf, read_sz); | |
452 | debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); | |
453 | if (len > 0) { | |
454 | fd_bytes(fd, len, FD_READ); | |
455 | #if DELAY_POOLS | |
456 | delayBytesIn(delay_id, len); | |
457 | #endif | |
458 | kb_incr(&Counter.server.all.kbytes_in, len); | |
459 | kb_incr(&Counter.server.http.kbytes_in, len); | |
460 | commSetTimeout(fd, Config.Timeout.read, NULL, NULL); | |
461 | IOStats.Http.reads++; | |
462 | for (clen = len - 1, bin = 0; clen; bin++) | |
463 | clen >>= 1; | |
464 | IOStats.Http.read_hist[bin]++; | |
465 | } | |
466 | if (!httpState->reply_hdr && len > 0) { | |
467 | /* Skip whitespace */ | |
468 | while (len > 0 && xisspace(*buf)) | |
469 | xmemmove(buf, buf + 1, len--); | |
470 | if (len == 0) { | |
471 | /* Continue to read... */ | |
472 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
473 | return; | |
474 | } | |
475 | } | |
476 | if (len < 0) { | |
477 | debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n", | |
478 | fd, xstrerror()); | |
479 | if (ignoreErrno(errno)) { | |
480 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
481 | } else if (entry->mem_obj->inmem_hi == 0) { | |
482 | ErrorState *err; | |
483 | err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); | |
484 | err->xerrno = errno; | |
485 | fwdFail(httpState->fwd, err); | |
486 | comm_close(fd); | |
487 | } else { | |
488 | comm_close(fd); | |
489 | } | |
490 | } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { | |
491 | ErrorState *err; | |
492 | err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); | |
493 | err->xerrno = errno; | |
494 | fwdFail(httpState->fwd, err); | |
495 | httpState->eof = 1; | |
496 | comm_close(fd); | |
497 | } else if (len == 0) { | |
498 | /* Connection closed; retrieval done. */ | |
499 | httpState->eof = 1; | |
500 | if (httpState->reply_hdr_state < 2) | |
501 | /* | |
502 | * Yes Henrik, there is a point to doing this. When we | |
503 | * called httpProcessReplyHeader() before, we didn't find | |
504 | * the end of headers, but now we are definately at EOF, so | |
505 | * we want to process the reply headers. | |
506 | */ | |
507 | httpProcessReplyHeader(httpState, buf, len); | |
508 | fwdComplete(httpState->fwd); | |
509 | comm_close(fd); | |
510 | } else { | |
511 | if (httpState->reply_hdr_state < 2) { | |
512 | httpProcessReplyHeader(httpState, buf, len); | |
513 | if (httpState->reply_hdr_state == 2) { | |
514 | http_status s = entry->mem_obj->reply->sline.status; | |
515 | /* | |
516 | * If its not a reply that we will re-forward, then | |
517 | * allow the client to get it. | |
518 | */ | |
519 | if (!fwdReforwardableStatus(s)) | |
520 | EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); | |
521 | } | |
522 | } | |
523 | storeAppend(entry, buf, len); | |
524 | if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { | |
525 | /* | |
526 | * the above storeAppend() call could ABORT this entry, | |
527 | * in that case, the server FD should already be closed. | |
528 | * there's nothing for us to do. | |
529 | */ | |
530 | (void) 0; | |
531 | } else if (httpPconnTransferDone(httpState)) { | |
532 | /* yes we have to clear all these! */ | |
533 | commSetDefer(fd, NULL, NULL); | |
534 | commSetTimeout(fd, -1, NULL, NULL); | |
535 | commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); | |
536 | #if DELAY_POOLS | |
537 | delayClearNoDelay(fd); | |
538 | #endif | |
539 | comm_remove_close_handler(fd, httpStateFree, httpState); | |
540 | fwdUnregister(fd, httpState->fwd); | |
541 | pconnPush(fd, request->host, request->port); | |
542 | fwdComplete(httpState->fwd); | |
543 | httpState->fd = -1; | |
544 | httpStateFree(fd, httpState); | |
545 | } else { | |
546 | /* Wait for EOF condition */ | |
547 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
548 | } | |
549 | } | |
550 | } | |
551 | ||
552 | /* This will be called when request write is complete. Schedule read of | |
553 | * reply. */ | |
554 | static void | |
555 | httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) | |
556 | { | |
557 | HttpStateData *httpState = data; | |
558 | StoreEntry *entry = httpState->entry; | |
559 | ErrorState *err; | |
560 | debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n", | |
561 | fd, size, errflag); | |
562 | #if URL_CHECKSUM_DEBUG | |
563 | assert(entry->mem_obj->chksum == url_checksum(entry->mem_obj->url)); | |
564 | #endif | |
565 | if (size > 0) { | |
566 | fd_bytes(fd, size, FD_WRITE); | |
567 | kb_incr(&Counter.server.all.kbytes_out, size); | |
568 | kb_incr(&Counter.server.http.kbytes_out, size); | |
569 | } | |
570 | if (errflag == COMM_ERR_CLOSING) | |
571 | return; | |
572 | if (errflag) { | |
573 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); | |
574 | err->xerrno = errno; | |
575 | err->request = requestLink(httpState->orig_request); | |
576 | errorAppendEntry(entry, err); | |
577 | comm_close(fd); | |
578 | return; | |
579 | } else { | |
580 | /* Schedule read reply. */ | |
581 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
582 | /* | |
583 | * Set the read timeout here because it hasn't been set yet. | |
584 | * We only set the read timeout after the request has been | |
585 | * fully written to the server-side. If we start the timeout | |
586 | * after connection establishment, then we are likely to hit | |
587 | * the timeout for POST/PUT requests that have very large | |
588 | * request bodies. | |
589 | */ | |
590 | commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); | |
591 | commSetDefer(fd, fwdCheckDeferRead, entry); | |
592 | } | |
593 | } | |
594 | ||
595 | /* | |
596 | * build request headers and append them to a given MemBuf | |
597 | * used by httpBuildRequestPrefix() | |
598 | * note: calls httpHeaderInit(), the caller is responsible for Clean()-ing | |
599 | */ | |
600 | void | |
601 | httpBuildRequestHeader(request_t * request, | |
602 | request_t * orig_request, | |
603 | StoreEntry * entry, | |
604 | HttpHeader * hdr_out, | |
605 | int cfd, | |
606 | http_state_flags flags) | |
607 | { | |
608 | /* building buffer for complex strings */ | |
609 | #define BBUF_SZ (MAX_URL+32) | |
610 | LOCAL_ARRAY(char, bbuf, BBUF_SZ); | |
611 | String strConnection = StringNull; | |
612 | const HttpHeader *hdr_in = &orig_request->header; | |
613 | int we_do_ranges; | |
614 | const HttpHeaderEntry *e; | |
615 | HttpHeaderPos pos = HttpHeaderInitPos; | |
616 | httpHeaderInit(hdr_out, hoRequest); | |
617 | /* append our IMS header */ | |
618 | if (request->lastmod > -1 && request->method == METHOD_GET) | |
619 | httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); | |
620 | ||
621 | /* decide if we want to do Ranges ourselves | |
622 | * (and fetch the whole object now) | |
623 | * We want to handle Ranges ourselves iff | |
624 | * - we can actually parse client Range specs | |
625 | * - the specs are expected to be simple enough (e.g. no out-of-order ranges) | |
626 | * - reply will be cachable | |
627 | * (If the reply will be uncachable we have to throw it away after | |
628 | * serving this request, so it is better to forward ranges to | |
629 | * the server and fetch only the requested content) | |
630 | */ | |
631 | we_do_ranges = | |
632 | orig_request->range && orig_request->flags.cachable && !httpHdrRangeWillBeComplex(orig_request->range) && (Config.rangeOffsetLimit == -1 || httpHdrRangeFirstOffset(orig_request->range) <= Config.rangeOffsetLimit); | |
633 | debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", | |
634 | orig_request->range, orig_request->flags.cachable, we_do_ranges); | |
635 | ||
636 | strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); | |
637 | while ((e = httpHeaderGetEntry(hdr_in, &pos))) { | |
638 | debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", | |
639 | strBuf(e->name), strBuf(e->value)); | |
640 | if (!httpRequestHdrAllowed(e, &strConnection)) | |
641 | continue; | |
642 | switch (e->id) { | |
643 | case HDR_PROXY_AUTHORIZATION: | |
644 | /* If we're not doing proxy auth, then it must be passed on */ | |
645 | if (!request->flags.used_proxy_auth) | |
646 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
647 | break; | |
648 | case HDR_AUTHORIZATION: | |
649 | /* If we're not doing www auth, then it must be passed on */ | |
650 | if (!request->flags.accelerated || !request->flags.used_proxy_auth) | |
651 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
652 | else | |
653 | request->flags.auth = 0; /* We have used the authentication */ | |
654 | break; | |
655 | case HDR_HOST: | |
656 | /* Don't use client's Host: header for redirected requests */ | |
657 | if (!request->flags.redirected || !Config.onoff.redir_rewrites_host) | |
658 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
659 | break; | |
660 | case HDR_IF_MODIFIED_SINCE: | |
661 | /* append unless we added our own; | |
662 | * note: at most one client's ims header can pass through */ | |
663 | if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) | |
664 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
665 | break; | |
666 | case HDR_MAX_FORWARDS: | |
667 | if (orig_request->method == METHOD_TRACE) { | |
668 | /* sacrificing efficiency over clarity, etc. */ | |
669 | const int hops = httpHeaderGetInt(hdr_in, HDR_MAX_FORWARDS); | |
670 | if (hops > 0) | |
671 | httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); | |
672 | } | |
673 | break; | |
674 | case HDR_RANGE: | |
675 | case HDR_IF_RANGE: | |
676 | case HDR_REQUEST_RANGE: | |
677 | if (!we_do_ranges) | |
678 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
679 | break; | |
680 | case HDR_PROXY_CONNECTION: | |
681 | case HDR_CONNECTION: | |
682 | case HDR_VIA: | |
683 | case HDR_X_FORWARDED_FOR: | |
684 | case HDR_CACHE_CONTROL: | |
685 | /* append these after the loop if needed */ | |
686 | break; | |
687 | default: | |
688 | /* pass on all other header fields */ | |
689 | httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); | |
690 | } | |
691 | } | |
692 | ||
693 | /* append fake user agent if configured and | |
694 | * the real one is not supplied by the client */ | |
695 | if (Config.fake_ua && !httpHeaderHas(hdr_out, HDR_USER_AGENT)) | |
696 | httpHeaderPutStr(hdr_out, HDR_USER_AGENT, Config.fake_ua); | |
697 | ||
698 | /* append Via */ | |
699 | { | |
700 | String strVia = httpHeaderGetList(hdr_in, HDR_VIA); | |
701 | snprintf(bbuf, BBUF_SZ, "%3.1f %s", orig_request->http_ver, ThisCache); | |
702 | strListAdd(&strVia, bbuf, ','); | |
703 | httpHeaderPutStr(hdr_out, HDR_VIA, strBuf(strVia)); | |
704 | stringClean(&strVia); | |
705 | } | |
706 | /* append X-Forwarded-For */ | |
707 | { | |
708 | String strFwd = httpHeaderGetList(hdr_in, HDR_X_FORWARDED_FOR); | |
709 | strListAdd(&strFwd, (cfd < 0 ? "unknown" : fd_table[cfd].ipaddr), ','); | |
710 | httpHeaderPutStr(hdr_out, HDR_X_FORWARDED_FOR, strBuf(strFwd)); | |
711 | stringClean(&strFwd); | |
712 | } | |
713 | /* append Host if not there already */ | |
714 | if (!httpHeaderHas(hdr_out, HDR_HOST)) { | |
715 | /* use port# only if not default */ | |
716 | if (orig_request->port == urlDefaultPort(orig_request->protocol)) { | |
717 | httpHeaderPutStr(hdr_out, HDR_HOST, orig_request->host); | |
718 | } else { | |
719 | httpHeaderPutStrf(hdr_out, HDR_HOST, "%s:%d", | |
720 | orig_request->host, (int) orig_request->port); | |
721 | } | |
722 | } | |
723 | /* append Authorization if known in URL, not in header and going direct */ | |
724 | if (!httpHeaderHas(hdr_out, HDR_AUTHORIZATION)) { | |
725 | if (!request->flags.proxying && *request->login) { | |
726 | httpHeaderPutStrf(hdr_out, HDR_AUTHORIZATION, "Basic %s", | |
727 | base64_encode(request->login)); | |
728 | } | |
729 | } | |
730 | /* append Proxy-Authorization if configured for peer, and proxying */ | |
731 | if (!httpHeaderHas(hdr_out, HDR_PROXY_AUTHORIZATION)) { | |
732 | if (request->flags.proxying && orig_request->peer_login) { | |
733 | httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", | |
734 | base64_encode(orig_request->peer_login)); | |
735 | } | |
736 | } | |
737 | /* append Cache-Control, add max-age if not there already */ | |
738 | { | |
739 | HttpHdrCc *cc = httpHeaderGetCc(hdr_in); | |
740 | if (!cc) | |
741 | cc = httpHdrCcCreate(); | |
742 | if (!EBIT_TEST(cc->mask, CC_MAX_AGE)) { | |
743 | const char *url = entry ? storeUrl(entry) : urlCanonical(orig_request); | |
744 | httpHdrCcSetMaxAge(cc, getMaxAge(url)); | |
745 | if (strLen(request->urlpath)) | |
746 | assert(strstr(url, strBuf(request->urlpath))); | |
747 | } | |
748 | if (flags.only_if_cached) | |
749 | EBIT_SET(cc->mask, CC_ONLY_IF_CACHED); | |
750 | httpHeaderPutCc(hdr_out, cc); | |
751 | httpHdrCcDestroy(cc); | |
752 | } | |
753 | /* maybe append Connection: keep-alive */ | |
754 | if (flags.keepalive) { | |
755 | if (flags.proxying) { | |
756 | httpHeaderPutStr(hdr_out, HDR_PROXY_CONNECTION, "keep-alive"); | |
757 | } else { | |
758 | httpHeaderPutStr(hdr_out, HDR_CONNECTION, "keep-alive"); | |
759 | } | |
760 | } | |
761 | stringClean(&strConnection); | |
762 | } | |
763 | ||
764 | /* build request prefix and append it to a given MemBuf; | |
765 | * return the length of the prefix */ | |
766 | mb_size_t | |
767 | httpBuildRequestPrefix(request_t * request, | |
768 | request_t * orig_request, | |
769 | StoreEntry * entry, | |
770 | MemBuf * mb, | |
771 | int cfd, | |
772 | http_state_flags flags) | |
773 | { | |
774 | const int offset = mb->size; | |
775 | memBufPrintf(mb, "%s %s HTTP/1.0\r\n", | |
776 | RequestMethodStr[request->method], | |
777 | strLen(request->urlpath) ? strBuf(request->urlpath) : "/"); | |
778 | /* build and pack headers */ | |
779 | { | |
780 | HttpHeader hdr; | |
781 | Packer p; | |
782 | httpBuildRequestHeader(request, orig_request, entry, &hdr, cfd, flags); | |
783 | packerToMemInit(&p, mb); | |
784 | httpHeaderPackInto(&hdr, &p); | |
785 | httpHeaderClean(&hdr); | |
786 | packerClean(&p); | |
787 | } | |
788 | /* append header terminator */ | |
789 | memBufAppend(mb, crlf, 2); | |
790 | return mb->size - offset; | |
791 | } | |
792 | /* This will be called when connect completes. Write request. */ | |
793 | static void | |
794 | httpSendRequest(HttpStateData * httpState) | |
795 | { | |
796 | MemBuf mb; | |
797 | request_t *req = httpState->request; | |
798 | StoreEntry *entry = httpState->entry; | |
799 | int cfd; | |
800 | peer *p = httpState->peer; | |
801 | CWCB *sendHeaderDone; | |
802 | ||
803 | debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", httpState->fd, httpState); | |
804 | ||
805 | if (httpState->orig_request->content_length > 0) | |
806 | sendHeaderDone = httpSendRequestEntry; | |
807 | else | |
808 | sendHeaderDone = httpSendComplete; | |
809 | ||
810 | if (!opt_forwarded_for) | |
811 | cfd = -1; | |
812 | else if (entry->mem_obj == NULL) | |
813 | cfd = -1; | |
814 | else | |
815 | cfd = entry->mem_obj->fd; | |
816 | assert(-1 == cfd || FD_SOCKET == fd_table[cfd].type); | |
817 | if (p != NULL) | |
818 | httpState->flags.proxying = 1; | |
819 | /* | |
820 | * Is keep-alive okay for all request methods? | |
821 | */ | |
822 | if (p == NULL) | |
823 | httpState->flags.keepalive = 1; | |
824 | else if (p->stats.n_keepalives_sent < 10) | |
825 | httpState->flags.keepalive = 1; | |
826 | else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) | |
827 | httpState->flags.keepalive = 1; | |
828 | if (httpState->peer) | |
829 | if (neighborType(httpState->peer, httpState->request) == PEER_SIBLING) | |
830 | httpState->flags.only_if_cached = 1; | |
831 | memBufDefInit(&mb); | |
832 | httpBuildRequestPrefix(req, | |
833 | httpState->orig_request, | |
834 | entry, | |
835 | &mb, | |
836 | cfd, | |
837 | httpState->flags); | |
838 | debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", httpState->fd, mb.buf); | |
839 | comm_write_mbuf(httpState->fd, mb, sendHeaderDone, httpState); | |
840 | } | |
841 | ||
842 | void | |
843 | httpStart(FwdState * fwd) | |
844 | { | |
845 | int fd = fwd->server_fd; | |
846 | HttpStateData *httpState = memAllocate(MEM_HTTP_STATE_DATA); | |
847 | request_t *proxy_req; | |
848 | request_t *orig_req = fwd->request; | |
849 | debug(11, 3) ("httpStart: \"%s %s\"\n", | |
850 | RequestMethodStr[orig_req->method], | |
851 | storeUrl(fwd->entry)); | |
852 | cbdataAdd(httpState, memFree, MEM_HTTP_STATE_DATA); | |
853 | storeLockObject(fwd->entry); | |
854 | httpState->fwd = fwd; | |
855 | httpState->entry = fwd->entry; | |
856 | httpState->fd = fd; | |
857 | if (fwd->servers) | |
858 | httpState->peer = fwd->servers->peer; /* might be NULL */ | |
859 | if (httpState->peer) { | |
860 | proxy_req = requestCreate(orig_req->method, | |
861 | PROTO_NONE, storeUrl(httpState->entry)); | |
862 | xstrncpy(proxy_req->host, httpState->peer->host, SQUIDHOSTNAMELEN); | |
863 | proxy_req->port = httpState->peer->http_port; | |
864 | proxy_req->flags = orig_req->flags; | |
865 | proxy_req->lastmod = orig_req->lastmod; | |
866 | httpState->request = requestLink(proxy_req); | |
867 | httpState->orig_request = requestLink(orig_req); | |
868 | proxy_req->flags.proxying = 1; | |
869 | /* | |
870 | * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here. | |
871 | * We might end up getting the object from somewhere else if, | |
872 | * for example, the request to this neighbor fails. | |
873 | */ | |
874 | if (httpState->peer->options.proxy_only) | |
875 | storeReleaseRequest(httpState->entry); | |
876 | #if DELAY_POOLS | |
877 | assert(delayIsNoDelay(fd) == 0); | |
878 | if (httpState->peer->options.no_delay) | |
879 | delaySetNoDelay(fd); | |
880 | #endif | |
881 | } else { | |
882 | httpState->request = requestLink(orig_req); | |
883 | httpState->orig_request = requestLink(orig_req); | |
884 | } | |
885 | /* | |
886 | * register the handler to free HTTP state data when the FD closes | |
887 | */ | |
888 | comm_add_close_handler(fd, httpStateFree, httpState); | |
889 | Counter.server.all.requests++; | |
890 | Counter.server.http.requests++; | |
891 | httpSendRequest(httpState); | |
892 | /* | |
893 | * We used to set the read timeout here, but not any more. | |
894 | * Now its set in httpSendComplete() after the full request, | |
895 | * including request body, has been written to the server. | |
896 | */ | |
897 | } | |
898 | ||
899 | static void | |
900 | httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void *data) | |
901 | { | |
902 | HttpStateData *httpState = data; | |
903 | StoreEntry *entry = httpState->entry; | |
904 | ErrorState *err; | |
905 | debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n", | |
906 | fd, size, errflag); | |
907 | if (size > 0) { | |
908 | fd_bytes(fd, size, FD_WRITE); | |
909 | kb_incr(&Counter.server.all.kbytes_out, size); | |
910 | kb_incr(&Counter.server.http.kbytes_out, size); | |
911 | } | |
912 | if (errflag == COMM_ERR_CLOSING) | |
913 | return; | |
914 | if (errflag) { | |
915 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); | |
916 | err->xerrno = errno; | |
917 | err->request = requestLink(httpState->orig_request); | |
918 | errorAppendEntry(entry, err); | |
919 | comm_close(fd); | |
920 | return; | |
921 | } | |
922 | if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { | |
923 | comm_close(fd); | |
924 | return; | |
925 | } | |
926 | pumpStart(fd, httpState->fwd, httpSendRequestEntryDone, httpState); | |
927 | } | |
928 | ||
929 | static void | |
930 | httpSendRequestEntryDone(int fd, char *bufnotused, size_t size, int errflag, void *data) | |
931 | { | |
932 | HttpStateData *httpState = data; | |
933 | StoreEntry *entry = httpState->entry; | |
934 | ErrorState *err; | |
935 | aclCheck_t ch; | |
936 | debug(11, 5) ("httpSendRequestEntryDone: FD %d: size %d: errflag %d.\n", | |
937 | fd, size, errflag); | |
938 | if (size > 0) { | |
939 | fd_bytes(fd, size, FD_WRITE); | |
940 | kb_incr(&Counter.server.all.kbytes_out, size); | |
941 | kb_incr(&Counter.server.http.kbytes_out, size); | |
942 | } | |
943 | if (errflag == COMM_ERR_CLOSING) | |
944 | return; | |
945 | if (errflag) { | |
946 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); | |
947 | err->xerrno = errno; | |
948 | err->request = requestLink(httpState->orig_request); | |
949 | errorAppendEntry(entry, err); | |
950 | comm_close(fd); | |
951 | return; | |
952 | } | |
953 | memset(&ch, '\0', sizeof(ch)); | |
954 | ch.request = httpState->request; | |
955 | if (!Config.accessList.brokenPosts) { | |
956 | debug(11, 5) ("httpSendRequestEntryDone: No brokenPosts list\n"); | |
957 | httpSendComplete(fd, NULL, 0, 0, data); | |
958 | } else if (!aclCheckFast(Config.accessList.brokenPosts, &ch)) { | |
959 | debug(11, 5) ("httpSendRequestEntryDone: didn't match brokenPosts\n"); | |
960 | httpSendComplete(fd, NULL, 0, 0, data); | |
961 | } else { | |
962 | debug(11, 2) ("httpSendRequestEntryDone: matched brokenPosts\n"); | |
963 | comm_write(fd, "\r\n", 2, httpSendComplete, data, NULL); | |
964 | } | |
965 | } |