]>
Commit | Line | Data |
---|---|---|
da2b3a17 | 1 | |
30a4f2a8 | 2 | /* |
a200bbd2 | 3 | * $Id: http.cc,v 1.235 1998/02/03 04:21:15 wessels Exp $ |
30a4f2a8 | 4 | * |
5 | * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) | |
6 | * AUTHOR: Harvest Derived | |
7 | * | |
42c04c16 | 8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
30a4f2a8 | 9 | * -------------------------------------------------------- |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
13 | * National Laboratory for Applied Network Research and funded by | |
14 | * the National Science Foundation. | |
15 | * | |
16 | * This program is free software; you can redistribute it and/or modify | |
17 | * it under the terms of the GNU General Public License as published by | |
18 | * the Free Software Foundation; either version 2 of the License, or | |
19 | * (at your option) any later version. | |
20 | * | |
21 | * This program is distributed in the hope that it will be useful, | |
22 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
24 | * GNU General Public License for more details. | |
25 | * | |
26 | * You should have received a copy of the GNU General Public License | |
27 | * along with this program; if not, write to the Free Software | |
28 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | * | |
30 | */ | |
019dd986 | 31 | |
32 | /* | |
30a4f2a8 | 33 | * Copyright (c) 1994, 1995. All rights reserved. |
34 | * | |
35 | * The Harvest software was developed by the Internet Research Task | |
36 | * Force Research Group on Resource Discovery (IRTF-RD): | |
37 | * | |
38 | * Mic Bowman of Transarc Corporation. | |
39 | * Peter Danzig of the University of Southern California. | |
40 | * Darren R. Hardy of the University of Colorado at Boulder. | |
41 | * Udi Manber of the University of Arizona. | |
42 | * Michael F. Schwartz of the University of Colorado at Boulder. | |
43 | * Duane Wessels of the University of Colorado at Boulder. | |
44 | * | |
45 | * This copyright notice applies to software in the Harvest | |
46 | * ``src/'' directory only. Users should consult the individual | |
47 | * copyright notices in the ``components/'' subdirectories for | |
48 | * copyright information about other software bundled with the | |
49 | * Harvest source code distribution. | |
50 | * | |
51 | * TERMS OF USE | |
52 | * | |
53 | * The Harvest software may be used and re-distributed without | |
54 | * charge, provided that the software origin and research team are | |
55 | * cited in any use of the system. Most commonly this is | |
56 | * accomplished by including a link to the Harvest Home Page | |
57 | * (http://harvest.cs.colorado.edu/) from the query page of any | |
58 | * Broker you deploy, as well as in the query result pages. These | |
59 | * links are generated automatically by the standard Broker | |
60 | * software distribution. | |
61 | * | |
62 | * The Harvest software is provided ``as is'', without express or | |
63 | * implied warranty, and with no support nor obligation to assist | |
64 | * in its use, correction, modification or enhancement. We assume | |
65 | * no liability with respect to the infringement of copyrights, | |
66 | * trade secrets, or any patents, and are not responsible for | |
67 | * consequential damages. Proper use of the Harvest software is | |
68 | * entirely the responsibility of the user. | |
69 | * | |
70 | * DERIVATIVE WORKS | |
71 | * | |
72 | * Users may make derivative works from the Harvest software, subject | |
73 | * to the following constraints: | |
74 | * | |
75 | * - You must include the above copyright notice and these | |
76 | * accompanying paragraphs in all forms of derivative works, | |
77 | * and any documentation and other materials related to such | |
78 | * distribution and use acknowledge that the software was | |
79 | * developed at the above institutions. | |
80 | * | |
81 | * - You must notify IRTF-RD regarding your distribution of | |
82 | * the derivative work. | |
83 | * | |
84 | * - You must clearly notify users that your are distributing | |
85 | * a modified version and not the original Harvest software. | |
86 | * | |
87 | * - Any derivative product is also subject to these copyright | |
88 | * and use restrictions. | |
89 | * | |
90 | * Note that the Harvest software is NOT in the public domain. We | |
91 | * retain copyright, as specified above. | |
92 | * | |
93 | * HISTORY OF FREE SOFTWARE STATUS | |
94 | * | |
95 | * Originally we required sites to license the software in cases | |
96 | * where they were going to build commercial products/services | |
97 | * around Harvest. In June 1995 we changed this policy. We now | |
98 | * allow people to use the core Harvest software (the code found in | |
99 | * the Harvest ``src/'' directory) for free. We made this change | |
100 | * in the interest of encouraging the widest possible deployment of | |
101 | * the technology. The Harvest software is really a reference | |
102 | * implementation of a set of protocols and formats, some of which | |
103 | * we intend to standardize. We encourage commercial | |
104 | * re-implementations of code complying to this set of standards. | |
019dd986 | 105 | */ |
44a47c6e | 106 | |
4a83b852 | 107 | /* |
108 | * Anonymizing patch by lutz@as-node.jena.thur.de | |
de3bdb4c | 109 | * have a look into http-anon.c to get more informations. |
4a83b852 | 110 | */ |
111 | ||
44a47c6e | 112 | #include "squid.h" |
090089c4 | 113 | |
6bf8443a | 114 | static const char *const crlf = "\r\n"; |
4db43fab | 115 | |
6fb52f6c | 116 | typedef enum { |
117 | SCC_PUBLIC, | |
118 | SCC_PRIVATE, | |
119 | SCC_NOCACHE, | |
120 | SCC_NOSTORE, | |
121 | SCC_NOTRANSFORM, | |
122 | SCC_MUSTREVALIDATE, | |
123 | SCC_PROXYREVALIDATE, | |
124 | SCC_MAXAGE, | |
125 | SCC_ENUM_END | |
126 | } http_server_cc_t; | |
127 | ||
6bf8443a | 128 | enum { |
6fb52f6c | 129 | CCC_NOCACHE, |
130 | CCC_NOSTORE, | |
131 | CCC_MAXAGE, | |
132 | CCC_MAXSTALE, | |
133 | CCC_MINFRESH, | |
134 | CCC_ONLYIFCACHED, | |
135 | CCC_ENUM_END | |
6bf8443a | 136 | }; |
137 | ||
151a0b6d | 138 | typedef enum { |
139 | HDR_ACCEPT, | |
140 | HDR_AGE, | |
141 | HDR_CONTENT_LENGTH, | |
142 | HDR_CONTENT_MD5, | |
143 | HDR_CONTENT_TYPE, | |
144 | HDR_DATE, | |
145 | HDR_ETAG, | |
146 | HDR_EXPIRES, | |
6bf8443a | 147 | HDR_HOST, |
151a0b6d | 148 | HDR_IMS, |
149 | HDR_LAST_MODIFIED, | |
150 | HDR_MAX_FORWARDS, | |
151 | HDR_PUBLIC, | |
152 | HDR_RETRY_AFTER, | |
153 | HDR_SET_COOKIE, | |
154 | HDR_UPGRADE, | |
155 | HDR_WARNING, | |
1294c0fc | 156 | HDR_PROXY_KEEPALIVE, |
151a0b6d | 157 | HDR_MISC_END |
158 | } http_hdr_misc_t; | |
6fb52f6c | 159 | |
e924600d | 160 | static char *HttpServerCCStr[] = |
6fb52f6c | 161 | { |
162 | "public", | |
163 | "private", | |
164 | "no-cache", | |
165 | "no-store", | |
166 | "no-transform", | |
167 | "must-revalidate", | |
168 | "proxy-revalidate", | |
169 | "max-age", | |
170 | "NONE" | |
171 | }; | |
172 | ||
151a0b6d | 173 | static char *HttpHdrMiscStr[] = |
174 | { | |
175 | "Accept", | |
176 | "Age", | |
177 | "Content-Length", | |
178 | "Content-MD5", | |
179 | "Content-Type", | |
180 | "Date", | |
181 | "Etag", | |
182 | "Expires", | |
183 | "Host", | |
184 | "If-Modified-Since", | |
185 | "Last-Modified", | |
186 | "Max-Forwards", | |
187 | "Public", | |
188 | "Retry-After", | |
189 | "Set-Cookie", | |
190 | "Upgrade", | |
191 | "Warning", | |
192 | "NONE" | |
193 | }; | |
194 | ||
24382924 | 195 | static struct { |
30a4f2a8 | 196 | int parsed; |
151a0b6d | 197 | int misc[HDR_MISC_END]; |
6fb52f6c | 198 | int cc[SCC_ENUM_END]; |
30a4f2a8 | 199 | } ReplyHeaderStats; |
090089c4 | 200 | |
9e4ad609 | 201 | static CNCB httpConnectDone; |
202 | static CWCB httpSendComplete; | |
9e4ad609 | 203 | static PF httpReadReply; |
204 | static PF httpSendRequest; | |
205 | static PF httpStateFree; | |
206 | static PF httpTimeout; | |
f5b8bbc4 | 207 | static void httpAppendRequestHeader(char *hdr, const char *line, size_t * sz, size_t max, int); |
208 | static void httpCacheNegatively(StoreEntry *); | |
209 | static void httpMakePrivate(StoreEntry *); | |
210 | static void httpMakePublic(StoreEntry *); | |
211 | static char *httpStatusString(int status); | |
bfcaf585 | 212 | static STABH httpAbort; |
f5b8bbc4 | 213 | static HttpStateData *httpBuildState(int, StoreEntry *, request_t *, peer *); |
214 | static int httpSocketOpen(StoreEntry *, request_t *); | |
215 | static void httpRestart(HttpStateData *); | |
f8309b15 | 216 | static int httpCachableReply(HttpStateData *); |
b8d8561b | 217 | |
b177367b | 218 | static void |
79d39a72 | 219 | httpStateFree(int fdnotused, void *data) |
f5558c95 | 220 | { |
b177367b | 221 | HttpStateData *httpState = data; |
0d4d4170 | 222 | if (httpState == NULL) |
b177367b | 223 | return; |
bfcaf585 | 224 | storeUnregisterAbort(httpState->entry); |
ddb6142d | 225 | assert(httpState->entry->store_status != STORE_PENDING); |
f88211e8 | 226 | storeUnlockObject(httpState->entry); |
0d4d4170 | 227 | if (httpState->reply_hdr) { |
3f6c0fb2 | 228 | memFree(MEM_8K_BUF, httpState->reply_hdr); |
0d4d4170 | 229 | httpState->reply_hdr = NULL; |
230 | } | |
30a4f2a8 | 231 | requestUnlink(httpState->request); |
20cc1450 | 232 | requestUnlink(httpState->orig_request); |
7dd44885 | 233 | httpState->request = NULL; |
234 | httpState->orig_request = NULL; | |
235 | cbdataFree(httpState); | |
f5558c95 | 236 | } |
237 | ||
b8d8561b | 238 | int |
75e88d56 | 239 | httpCachable(method_t method) |
090089c4 | 240 | { |
090089c4 | 241 | /* GET and HEAD are cachable. Others are not. */ |
6eb42cae | 242 | if (method != METHOD_GET && method != METHOD_HEAD) |
090089c4 | 243 | return 0; |
090089c4 | 244 | /* else cachable */ |
245 | return 1; | |
246 | } | |
247 | ||
b8d8561b | 248 | static void |
5c5783a2 | 249 | httpTimeout(int fd, void *data) |
090089c4 | 250 | { |
b177367b | 251 | HttpStateData *httpState = data; |
593c9a75 | 252 | StoreEntry *entry = httpState->entry; |
9b312a19 | 253 | ErrorState *err; |
9fb13bb6 | 254 | debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); |
8796b9e9 | 255 | assert(entry->store_status == STORE_PENDING); |
73a3014d | 256 | if (entry->mem_obj->inmem_hi == 0) { |
fe40a877 | 257 | err = errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT); |
79a15e0a | 258 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 259 | errorAppendEntry(entry, err); |
b50179a6 | 260 | } else { |
b34ed725 | 261 | storeAbort(entry, 0); |
9b312a19 | 262 | } |
0d4d4170 | 263 | comm_close(fd); |
090089c4 | 264 | } |
265 | ||
30a4f2a8 | 266 | /* This object can be cached for a long time */ |
b8d8561b | 267 | static void |
268 | httpMakePublic(StoreEntry * entry) | |
30a4f2a8 | 269 | { |
79a15e0a | 270 | if (EBIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 271 | storeSetPublicKey(entry); |
272 | } | |
273 | ||
274 | /* This object should never be cached at all */ | |
b8d8561b | 275 | static void |
276 | httpMakePrivate(StoreEntry * entry) | |
30a4f2a8 | 277 | { |
30a4f2a8 | 278 | storeExpireNow(entry); |
79a15e0a | 279 | EBIT_CLR(entry->flag, ENTRY_CACHABLE); |
30a4f2a8 | 280 | storeReleaseRequest(entry); /* delete object when not used */ |
281 | } | |
282 | ||
283 | /* This object may be negatively cached */ | |
b8d8561b | 284 | static void |
285 | httpCacheNegatively(StoreEntry * entry) | |
30a4f2a8 | 286 | { |
79b5cc5f | 287 | storeNegativeCache(entry); |
79a15e0a | 288 | if (EBIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 289 | storeSetPublicKey(entry); |
30a4f2a8 | 290 | } |
291 | ||
292 | ||
293 | /* Build a reply structure from HTTP reply headers */ | |
b8d8561b | 294 | void |
48f44632 | 295 | httpParseReplyHeaders(const char *buf, struct _http_reply *reply) |
30a4f2a8 | 296 | { |
3f6c0fb2 | 297 | char *headers = memAllocate(MEM_4K_BUF, 1); |
ca85027a | 298 | char *line; |
33b589ff | 299 | char *end; |
30a4f2a8 | 300 | char *s = NULL; |
33b589ff | 301 | char *t; |
ca98227c | 302 | time_t delta; |
303 | size_t l; | |
30a4f2a8 | 304 | |
5b29969a | 305 | assert(reply != NULL); |
f1494beb | 306 | reply->code = 600; |
30a4f2a8 | 307 | ReplyHeaderStats.parsed++; |
33b589ff | 308 | xstrncpy(headers, buf, 4096); |
309 | end = mime_headers_end(headers); | |
ca85027a | 310 | if (end == NULL) { |
311 | t = headers; | |
e2ad7f85 | 312 | if (!strncasecmp(t, "HTTP/", 5)) { |
313 | reply->version = atof(t + 5); | |
314 | if ((t = strchr(t, ' '))) | |
315 | reply->code = atoi(++t); | |
ca85027a | 316 | } |
3f6c0fb2 | 317 | memFree(MEM_4K_BUF, headers); |
e2ad7f85 | 318 | return; |
ca85027a | 319 | } |
320 | reply->hdr_sz = end - headers; | |
3f6c0fb2 | 321 | line = memAllocate(MEM_4K_BUF, 1); |
33b589ff | 322 | for (s = headers; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { |
323 | l = strcspn(s, crlf) + 1; | |
324 | if (l > 4096) | |
325 | l = 4096; | |
326 | xstrncpy(line, s, l); | |
327 | t = line; | |
a3d5953d | 328 | debug(11, 3) ("httpParseReplyHeaders: %s\n", t); |
33b589ff | 329 | if (!strncasecmp(t, "HTTP/", 5)) { |
e2ad7f85 | 330 | reply->version = atof(t + 5); |
33b589ff | 331 | if ((t = strchr(t, ' '))) |
332 | reply->code = atoi(++t); | |
30a4f2a8 | 333 | } else if (!strncasecmp(t, "Content-type:", 13)) { |
33b589ff | 334 | for (t += 13; isspace(*t); t++); |
e91b56e5 | 335 | if ((l = strcspn(t, ";\t ")) > 0) |
336 | *(t + l) = '\0'; | |
33b589ff | 337 | xstrncpy(reply->content_type, t, HTTP_REPLY_FIELD_SZ); |
151a0b6d | 338 | ReplyHeaderStats.misc[HDR_CONTENT_TYPE]++; |
30a4f2a8 | 339 | } else if (!strncasecmp(t, "Content-length:", 15)) { |
33b589ff | 340 | for (t += 15; isspace(*t); t++); |
33b589ff | 341 | reply->content_length = atoi(t); |
151a0b6d | 342 | ReplyHeaderStats.misc[HDR_CONTENT_LENGTH]++; |
30a4f2a8 | 343 | } else if (!strncasecmp(t, "Date:", 5)) { |
33b589ff | 344 | for (t += 5; isspace(*t); t++); |
345 | reply->date = parse_rfc1123(t); | |
151a0b6d | 346 | ReplyHeaderStats.misc[HDR_DATE]++; |
30a4f2a8 | 347 | } else if (!strncasecmp(t, "Expires:", 8)) { |
33b589ff | 348 | for (t += 8; isspace(*t); t++); |
349 | reply->expires = parse_rfc1123(t); | |
350 | /* | |
351 | * The HTTP/1.0 specs says that robust implementations | |
352 | * should consider bad or malformed Expires header as | |
353 | * equivalent to "expires immediately." | |
354 | */ | |
355 | if (reply->expires == -1) | |
356 | reply->expires = squid_curtime; | |
151a0b6d | 357 | ReplyHeaderStats.misc[HDR_EXPIRES]++; |
30a4f2a8 | 358 | } else if (!strncasecmp(t, "Last-Modified:", 14)) { |
33b589ff | 359 | for (t += 14; isspace(*t); t++); |
360 | reply->last_modified = parse_rfc1123(t); | |
151a0b6d | 361 | ReplyHeaderStats.misc[HDR_LAST_MODIFIED]++; |
362 | } else if (!strncasecmp(t, "Accept:", 7)) { | |
363 | ReplyHeaderStats.misc[HDR_ACCEPT]++; | |
364 | } else if (!strncasecmp(t, "Age:", 4)) { | |
365 | ReplyHeaderStats.misc[HDR_AGE]++; | |
366 | } else if (!strncasecmp(t, "Content-MD5:", 12)) { | |
367 | ReplyHeaderStats.misc[HDR_CONTENT_MD5]++; | |
368 | } else if (!strncasecmp(t, "ETag:", 5)) { | |
369 | ReplyHeaderStats.misc[HDR_ETAG]++; | |
370 | } else if (!strncasecmp(t, "Max-Forwards:", 13)) { | |
371 | ReplyHeaderStats.misc[HDR_MAX_FORWARDS]++; | |
372 | } else if (!strncasecmp(t, "Public:", 7)) { | |
373 | ReplyHeaderStats.misc[HDR_PUBLIC]++; | |
374 | } else if (!strncasecmp(t, "Retry-After:", 12)) { | |
375 | ReplyHeaderStats.misc[HDR_RETRY_AFTER]++; | |
376 | } else if (!strncasecmp(t, "Upgrade:", 8)) { | |
377 | ReplyHeaderStats.misc[HDR_UPGRADE]++; | |
378 | } else if (!strncasecmp(t, "Warning:", 8)) { | |
379 | ReplyHeaderStats.misc[HDR_WARNING]++; | |
caebbe00 | 380 | } else if (!strncasecmp(t, "Cache-Control:", 14)) { |
33b589ff | 381 | for (t += 14; isspace(*t); t++); |
4db43fab | 382 | if (!strncasecmp(t, "public", 6)) { |
383 | EBIT_SET(reply->cache_control, SCC_PUBLIC); | |
384 | ReplyHeaderStats.cc[SCC_PUBLIC]++; | |
385 | } else if (!strncasecmp(t, "private", 7)) { | |
386 | EBIT_SET(reply->cache_control, SCC_PRIVATE); | |
387 | ReplyHeaderStats.cc[SCC_PRIVATE]++; | |
388 | } else if (!strncasecmp(t, "no-cache", 8)) { | |
389 | EBIT_SET(reply->cache_control, SCC_NOCACHE); | |
390 | ReplyHeaderStats.cc[SCC_NOCACHE]++; | |
c1764328 | 391 | } else if (!strncasecmp(t, "no-store", 8)) { |
392 | EBIT_SET(reply->cache_control, SCC_NOSTORE); | |
393 | ReplyHeaderStats.cc[SCC_NOSTORE]++; | |
394 | } else if (!strncasecmp(t, "no-transform", 12)) { | |
395 | EBIT_SET(reply->cache_control, SCC_NOTRANSFORM); | |
396 | ReplyHeaderStats.cc[SCC_NOTRANSFORM]++; | |
397 | } else if (!strncasecmp(t, "must-revalidate", 15)) { | |
398 | EBIT_SET(reply->cache_control, SCC_MUSTREVALIDATE); | |
399 | ReplyHeaderStats.cc[SCC_MUSTREVALIDATE]++; | |
400 | } else if (!strncasecmp(t, "proxy-revalidate", 16)) { | |
401 | EBIT_SET(reply->cache_control, SCC_PROXYREVALIDATE); | |
402 | ReplyHeaderStats.cc[SCC_PROXYREVALIDATE]++; | |
4db43fab | 403 | } else if (!strncasecmp(t, "max-age", 7)) { |
404 | if ((t = strchr(t, '='))) { | |
ca98227c | 405 | delta = (time_t) atoi(++t); |
406 | reply->expires = squid_curtime + delta; | |
4db43fab | 407 | EBIT_SET(reply->cache_control, SCC_MAXAGE); |
408 | ReplyHeaderStats.cc[SCC_MAXAGE]++; | |
5df61230 | 409 | } |
caebbe00 | 410 | } |
df3ac7c0 | 411 | } else if (!strncasecmp(t, "Set-Cookie:", 11)) { |
412 | EBIT_SET(reply->misc_headers, HDR_SET_COOKIE); | |
58202be7 | 413 | ReplyHeaderStats.misc[HDR_SET_COOKIE]++; |
1294c0fc | 414 | } else if (!strncasecmp(t, "Proxy-Connection:", 17)) { |
537eedcb | 415 | for (t += 17; isspace(*t); t++); |
1294c0fc | 416 | if (!strcasecmp(t, "Keep-Alive")) |
417 | EBIT_SET(reply->misc_headers, HDR_PROXY_KEEPALIVE); | |
30a4f2a8 | 418 | } |
30a4f2a8 | 419 | } |
3f6c0fb2 | 420 | memFree(MEM_4K_BUF, headers); |
421 | memFree(MEM_4K_BUF, line); | |
30a4f2a8 | 422 | } |
423 | ||
f8309b15 | 424 | static int |
425 | httpCachableReply(HttpStateData * httpState) | |
c54e9052 | 426 | { |
f8309b15 | 427 | struct _http_reply *reply = httpState->entry->mem_obj->reply; |
428 | if (EBIT_TEST(reply->cache_control, SCC_PRIVATE)) | |
429 | return 0; | |
430 | if (EBIT_TEST(reply->cache_control, SCC_NOCACHE)) | |
431 | return 0; | |
79a15e0a | 432 | if (EBIT_TEST(httpState->request->flags, REQ_AUTH)) |
fee0cebb | 433 | if (!EBIT_TEST(reply->cache_control, SCC_PROXYREVALIDATE)) |
434 | return 0; | |
f8309b15 | 435 | /* |
436 | * Dealing with cookies is quite a bit more complicated | |
437 | * than this. Ideally we should strip the cookie | |
438 | * header from the reply but still cache the reply body. | |
439 | * More confusion at draft-ietf-http-state-mgmt-05.txt. | |
440 | */ | |
441 | if (EBIT_TEST(reply->misc_headers, HDR_SET_COOKIE)) | |
442 | return 0; | |
c54e9052 | 443 | switch (reply->code) { |
444 | /* Responses that are cacheable */ | |
445 | case 200: /* OK */ | |
446 | case 203: /* Non-Authoritative Information */ | |
447 | case 300: /* Multiple Choices */ | |
448 | case 301: /* Moved Permanently */ | |
449 | case 410: /* Gone */ | |
1294c0fc | 450 | /* don't cache objects from peers w/o LMT, Date, or Expires */ |
c54e9052 | 451 | if (reply->date > -1) |
452 | return 1; | |
f8309b15 | 453 | else if (reply->last_modified > -1) |
c54e9052 | 454 | return 1; |
1294c0fc | 455 | else if (!httpState->peer) |
c54e9052 | 456 | return 1; |
f8309b15 | 457 | else if (reply->expires > -1) |
c54e9052 | 458 | return 1; |
c54e9052 | 459 | else |
460 | return 0; | |
79d39a72 | 461 | /* NOTREACHED */ |
c54e9052 | 462 | break; |
463 | /* Responses that only are cacheable if the server says so */ | |
464 | case 302: /* Moved temporarily */ | |
465 | if (reply->expires > -1) | |
466 | return 1; | |
467 | else | |
468 | return 0; | |
79d39a72 | 469 | /* NOTREACHED */ |
c54e9052 | 470 | break; |
471 | /* Errors can be negatively cached */ | |
472 | case 204: /* No Content */ | |
473 | case 305: /* Use Proxy (proxy redirect) */ | |
474 | case 400: /* Bad Request */ | |
475 | case 403: /* Forbidden */ | |
476 | case 404: /* Not Found */ | |
477 | case 405: /* Method Now Allowed */ | |
478 | case 414: /* Request-URI Too Long */ | |
479 | case 500: /* Internal Server Error */ | |
480 | case 501: /* Not Implemented */ | |
481 | case 502: /* Bad Gateway */ | |
482 | case 503: /* Service Unavailable */ | |
483 | case 504: /* Gateway Timeout */ | |
484 | return -1; | |
79d39a72 | 485 | /* NOTREACHED */ |
c54e9052 | 486 | break; |
487 | /* Some responses can never be cached */ | |
88738790 | 488 | case 206: /* Partial Content -- Not yet supported */ |
c54e9052 | 489 | case 303: /* See Other */ |
490 | case 304: /* Not Modified */ | |
491 | case 401: /* Unauthorized */ | |
492 | case 407: /* Proxy Authentication Required */ | |
493 | case 600: /* Squid header parsing error */ | |
494 | default: /* Unknown status code */ | |
495 | return 0; | |
79d39a72 | 496 | /* NOTREACHED */ |
c54e9052 | 497 | break; |
498 | } | |
79d39a72 | 499 | /* NOTREACHED */ |
c54e9052 | 500 | } |
090089c4 | 501 | |
b8d8561b | 502 | void |
0ee4272b | 503 | httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) |
f5558c95 | 504 | { |
505 | char *t = NULL; | |
30a4f2a8 | 506 | StoreEntry *entry = httpState->entry; |
d3fb4dea | 507 | int room; |
508 | int hdr_len; | |
33b589ff | 509 | struct _http_reply *reply = entry->mem_obj->reply; |
b6cfb65c | 510 | debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", |
511 | storeKeyText(entry->key)); | |
e924600d | 512 | if (httpState->reply_hdr == NULL) |
3f6c0fb2 | 513 | httpState->reply_hdr = memAllocate(MEM_8K_BUF, 1); |
30a4f2a8 | 514 | if (httpState->reply_hdr_state == 0) { |
515 | hdr_len = strlen(httpState->reply_hdr); | |
ed85b771 | 516 | room = 8191 - hdr_len; |
30a4f2a8 | 517 | strncat(httpState->reply_hdr, buf, room < size ? room : size); |
d3fb4dea | 518 | hdr_len += room < size ? room : size; |
30a4f2a8 | 519 | if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { |
84fa351c | 520 | debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr); |
30a4f2a8 | 521 | httpState->reply_hdr_state += 2; |
33b589ff | 522 | reply->code = 555; |
ed85b771 | 523 | return; |
d3fb4dea | 524 | } |
d1a43e28 | 525 | t = httpState->reply_hdr + hdr_len; |
526 | /* headers can be incomplete only if object still arriving */ | |
f86a6a46 | 527 | if (!httpState->eof) |
d1a43e28 | 528 | if ((t = mime_headers_end(httpState->reply_hdr)) == NULL) |
529 | return; /* headers not complete */ | |
2285407f | 530 | *t = '\0'; |
30a4f2a8 | 531 | httpState->reply_hdr_state++; |
f5558c95 | 532 | } |
30a4f2a8 | 533 | if (httpState->reply_hdr_state == 1) { |
534 | httpState->reply_hdr_state++; | |
a3d5953d | 535 | debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", |
30a4f2a8 | 536 | httpState->reply_hdr); |
537 | /* Parse headers into reply structure */ | |
48f44632 | 538 | httpParseReplyHeaders(httpState->reply_hdr, reply); |
ca98227c | 539 | storeTimestampsSet(entry); |
30a4f2a8 | 540 | /* Check if object is cacheable or not based on reply code */ |
a3d5953d | 541 | debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->code); |
f8309b15 | 542 | switch (httpCachableReply(httpState)) { |
c54e9052 | 543 | case 1: |
544 | httpMakePublic(entry); | |
30a4f2a8 | 545 | break; |
c54e9052 | 546 | case 0: |
547 | httpMakePrivate(entry); | |
f5558c95 | 548 | break; |
c54e9052 | 549 | case -1: |
851eeef7 | 550 | httpCacheNegatively(entry); |
30a4f2a8 | 551 | break; |
c54e9052 | 552 | default: |
553 | assert(0); | |
4e38e700 | 554 | break; |
f5558c95 | 555 | } |
c54e9052 | 556 | if (EBIT_TEST(reply->cache_control, SCC_PROXYREVALIDATE)) |
79a15e0a | 557 | EBIT_SET(entry->flag, ENTRY_REVALIDATE); |
9a47da71 | 558 | if (EBIT_TEST(httpState->flags, HTTP_KEEPALIVE)) |
559 | if (httpState->peer) | |
560 | httpState->peer->stats.n_keepalives_sent++; | |
1294c0fc | 561 | if (EBIT_TEST(reply->misc_headers, HDR_PROXY_KEEPALIVE)) |
562 | if (httpState->peer) | |
563 | httpState->peer->stats.n_keepalives_recv++; | |
f5558c95 | 564 | } |
565 | } | |
566 | ||
603a02fd | 567 | static int |
568 | httpPconnTransferDone(HttpStateData * httpState) | |
569 | { | |
570 | /* return 1 if we got the last of the data on a persistent connection */ | |
571 | MemObject *mem = httpState->entry->mem_obj; | |
572 | struct _http_reply *reply = mem->reply; | |
51fdcbd5 | 573 | debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); |
978e455f | 574 | /* |
575 | * If we didn't send a Keepalive request header, then this | |
576 | * can not be a persistent connection. | |
577 | */ | |
79a15e0a | 578 | if (!EBIT_TEST(httpState->flags, HTTP_KEEPALIVE)) |
603a02fd | 579 | return 0; |
51fdcbd5 | 580 | debug(11, 5) ("httpPconnTransferDone: content_length=%d\n", |
603a02fd | 581 | reply->content_length); |
582 | /* | |
978e455f | 583 | * Deal with gross HTTP stuff |
584 | * - If we haven't seen the end of the reply headers, we can't | |
585 | * be persistent. | |
586 | * - For "200 OK" check the content-length in the next block. | |
978e455f | 587 | * - For "204 No Content" (even with content-length) we're done. |
588 | * - For "304 Not Modified" (even with content-length) we're done. | |
a3c60429 | 589 | * - 1XX replies never have a body; we're done. |
978e455f | 590 | * - For HEAD requests with content-length we're done. |
a3c60429 | 591 | * - For all other replies, check content length in next block. |
603a02fd | 592 | */ |
978e455f | 593 | if (httpState->reply_hdr_state < 2) |
594 | return 0; | |
595 | else if (reply->code == HTTP_OK) | |
a3c60429 | 596 | (void) 0; /* common case, continue */ |
978e455f | 597 | else if (reply->code == HTTP_NO_CONTENT) |
598 | return 1; | |
599 | else if (reply->code == HTTP_NOT_MODIFIED) | |
600 | return 1; | |
a3c60429 | 601 | else if (reply->code < HTTP_OK) |
602 | return 1; | |
978e455f | 603 | else if (httpState->request->method == METHOD_HEAD) |
604 | return 1; | |
603a02fd | 605 | /* |
a3c60429 | 606 | * If there is no content-length, then we can't be |
978e455f | 607 | * persistent. If there is a content length, then we must |
608 | * wait until we've seen the end of the body. | |
603a02fd | 609 | */ |
1790d392 | 610 | if (reply->content_length < 0) |
603a02fd | 611 | return 0; |
978e455f | 612 | else if (mem->inmem_hi < reply->content_length + reply->hdr_sz) |
603a02fd | 613 | return 0; |
978e455f | 614 | else |
b34ed725 | 615 | return 1; |
603a02fd | 616 | } |
090089c4 | 617 | |
618 | /* This will be called when data is ready to be read from fd. Read until | |
619 | * error or connection closed. */ | |
f5558c95 | 620 | /* XXX this function is too long! */ |
b8d8561b | 621 | static void |
b177367b | 622 | httpReadReply(int fd, void *data) |
090089c4 | 623 | { |
b177367b | 624 | HttpStateData *httpState = data; |
95d659f0 | 625 | LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); |
bfcaf585 | 626 | StoreEntry *entry = httpState->entry; |
603a02fd | 627 | const request_t *request = httpState->request; |
090089c4 | 628 | int len; |
30a4f2a8 | 629 | int bin; |
090089c4 | 630 | int clen; |
9b312a19 | 631 | ErrorState *err; |
d89d1fb6 | 632 | if (protoAbortFetch(entry)) { |
9b312a19 | 633 | storeAbort(entry, 0); |
a3d5953d | 634 | comm_close(fd); |
635 | return; | |
234967c9 | 636 | } |
637 | /* check if we want to defer reading */ | |
8350fe9b | 638 | clen = entry->mem_obj->inmem_hi; |
1513873c | 639 | errno = 0; |
30a4f2a8 | 640 | len = read(fd, buf, SQUID_TCP_SO_RCVBUF); |
4f92c80c | 641 | fd_bytes(fd, len, FD_READ); |
a3d5953d | 642 | debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); |
30a4f2a8 | 643 | if (len > 0) { |
4f92c80c | 644 | commSetTimeout(fd, Config.Timeout.read, NULL, NULL); |
4a63c85f | 645 | IOStats.Http.reads++; |
30a4f2a8 | 646 | for (clen = len - 1, bin = 0; clen; bin++) |
647 | clen >>= 1; | |
648 | IOStats.Http.read_hist[bin]++; | |
649 | } | |
ba718c8f | 650 | if (len < 0) { |
b224ea98 | 651 | if (ignoreErrno(errno)) { |
9b312a19 | 652 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); |
090089c4 | 653 | } else { |
73a3014d | 654 | if (clen == 0) { |
fe40a877 | 655 | err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); |
c45ed9ad | 656 | err->xerrno = errno; |
79a15e0a | 657 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 658 | errorAppendEntry(entry, err); |
b50179a6 | 659 | } else { |
b34ed725 | 660 | storeAbort(entry, 0); |
9b312a19 | 661 | } |
0d4d4170 | 662 | comm_close(fd); |
090089c4 | 663 | } |
a3d5953d | 664 | debug(50, 2) ("httpReadReply: FD %d: read failure: %s.\n", |
0a0bf5db | 665 | fd, xstrerror()); |
8350fe9b | 666 | } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { |
b716a8ad | 667 | if (fd_table[fd].uses > 1) { |
668 | httpRestart(httpState); | |
669 | } else { | |
670 | httpState->eof = 1; | |
fe40a877 | 671 | err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); |
c45ed9ad | 672 | err->xerrno = errno; |
79a15e0a | 673 | err->request = requestLink(httpState->orig_request); |
b716a8ad | 674 | errorAppendEntry(entry, err); |
b716a8ad | 675 | comm_close(fd); |
676 | } | |
090089c4 | 677 | } else if (len == 0) { |
678 | /* Connection closed; retrieval done. */ | |
f86a6a46 | 679 | httpState->eof = 1; |
d1a43e28 | 680 | if (httpState->reply_hdr_state < 2) |
b34ed725 | 681 | /* |
682 | * Yes Henrik, there is a point to doing this. When we | |
683 | * called httpProcessReplyHeader() before, we didn't find | |
684 | * the end of headers, but now we are definately at EOF, so | |
685 | * we want to process the reply headers. | |
686 | */ | |
d1a43e28 | 687 | httpProcessReplyHeader(httpState, buf, len); |
d1a43e28 | 688 | storeComplete(entry); /* deallocates mem_obj->request */ |
0d4d4170 | 689 | comm_close(fd); |
090089c4 | 690 | } else { |
d1a43e28 | 691 | if (httpState->reply_hdr_state < 2) |
30a4f2a8 | 692 | httpProcessReplyHeader(httpState, buf, len); |
620da955 | 693 | storeAppend(entry, buf, len); |
603a02fd | 694 | if (httpPconnTransferDone(httpState)) { |
5b29969a | 695 | /* yes we have to clear all these! */ |
8796b9e9 | 696 | commSetDefer(fd, NULL, NULL); |
5b29969a | 697 | commSetTimeout(fd, -1, NULL, NULL); |
698 | commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); | |
603a02fd | 699 | comm_remove_close_handler(fd, httpStateFree, httpState); |
700 | storeComplete(entry); /* deallocates mem_obj->request */ | |
8796b9e9 | 701 | pconnPush(fd, request->host, request->port); |
603a02fd | 702 | httpState->fd = -1; |
703 | httpStateFree(-1, httpState); | |
704 | } else { | |
705 | commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); | |
706 | } | |
090089c4 | 707 | } |
708 | } | |
709 | ||
710 | /* This will be called when request write is complete. Schedule read of | |
711 | * reply. */ | |
b8d8561b | 712 | static void |
79a15e0a | 713 | httpSendComplete(int fd, char *bufnotused, size_t size, int errflag, void *data) |
090089c4 | 714 | { |
30a4f2a8 | 715 | HttpStateData *httpState = data; |
9b312a19 | 716 | StoreEntry *entry = httpState->entry; |
717 | ErrorState *err; | |
a3d5953d | 718 | debug(11, 5) ("httpSendComplete: FD %d: size %d: errflag %d.\n", |
090089c4 | 719 | fd, size, errflag); |
ea3a2a69 | 720 | if (errflag == COMM_ERR_CLOSING) |
721 | return; | |
090089c4 | 722 | if (errflag) { |
fe40a877 | 723 | err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR); |
c45ed9ad | 724 | err->xerrno = errno; |
79a15e0a | 725 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 726 | errorAppendEntry(entry, err); |
0d4d4170 | 727 | comm_close(fd); |
090089c4 | 728 | return; |
729 | } else { | |
730 | /* Schedule read reply. */ | |
b177367b | 731 | commSetSelect(fd, |
019dd986 | 732 | COMM_SELECT_READ, |
b177367b | 733 | httpReadReply, |
cd1fb0eb | 734 | httpState, 0); |
70a9dab4 | 735 | commSetDefer(fd, protoCheckDeferRead, entry); |
090089c4 | 736 | } |
737 | } | |
738 | ||
6bf8443a | 739 | static void |
88738790 | 740 | httpAppendRequestHeader(char *hdr, const char *line, size_t * sz, size_t max, int check) |
6bf8443a | 741 | { |
742 | size_t n = *sz + strlen(line) + 2; | |
743 | if (n >= max) | |
744 | return; | |
88738790 | 745 | if (check) { |
17a0a4ee | 746 | if (Config.onoff.anonymizer == ANONYMIZER_PARANOID) { |
88738790 | 747 | if (!httpAnonAllowed(line)) |
748 | return; | |
17a0a4ee | 749 | } else if (Config.onoff.anonymizer == ANONYMIZER_STANDARD) { |
88738790 | 750 | if (httpAnonDenied(line)) |
751 | return; | |
752 | } | |
4a83b852 | 753 | } |
4a83b852 | 754 | /* allowed header, explicitly known to be not dangerous */ |
a3d5953d | 755 | debug(11, 5) ("httpAppendRequestHeader: %s\n", line); |
929545fe | 756 | strcpy(hdr + (*sz), line); |
6bf8443a | 757 | strcat(hdr + (*sz), crlf); |
758 | *sz = n; | |
759 | } | |
760 | ||
1294c0fc | 761 | #define YBUF_SZ (MAX_URL+32) |
6bf8443a | 762 | size_t |
763 | httpBuildRequestHeader(request_t * request, | |
764 | request_t * orig_request, | |
765 | StoreEntry * entry, | |
6bf8443a | 766 | size_t * in_len, |
767 | char *hdr_out, | |
768 | size_t out_sz, | |
603a02fd | 769 | int cfd, |
770 | int flags) | |
6bf8443a | 771 | { |
1294c0fc | 772 | LOCAL_ARRAY(char, ybuf, YBUF_SZ); |
ab013258 | 773 | LOCAL_ARRAY(char, no_forward, 1024); |
3f6c0fb2 | 774 | char *xbuf = memAllocate(MEM_4K_BUF, 1); |
775 | char *viabuf = memAllocate(MEM_4K_BUF, 1); | |
776 | char *fwdbuf = memAllocate(MEM_4K_BUF, 1); | |
6bf8443a | 777 | char *t = NULL; |
778 | char *s = NULL; | |
779 | char *end = NULL; | |
780 | size_t len = 0; | |
781 | size_t hdr_len = 0; | |
6bf8443a | 782 | size_t l; |
783 | int hdr_flags = 0; | |
151a0b6d | 784 | int cc_flags = 0; |
b3b64e58 | 785 | int n; |
6bf8443a | 786 | const char *url = NULL; |
2357f74a | 787 | char *hdr_in = orig_request->headers; |
6bf8443a | 788 | |
2357f74a | 789 | assert(hdr_in != NULL); |
a3d5953d | 790 | debug(11, 3) ("httpBuildRequestHeader: INPUT:\n%s\n", hdr_in); |
6bf8443a | 791 | xstrncpy(fwdbuf, "X-Forwarded-For: ", 4096); |
792 | xstrncpy(viabuf, "Via: ", 4096); | |
1294c0fc | 793 | snprintf(ybuf, YBUF_SZ, "%s %s HTTP/1.0", |
6bf8443a | 794 | RequestMethodStr[request->method], |
795 | *request->urlpath ? request->urlpath : "/"); | |
88738790 | 796 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 1); |
6bf8443a | 797 | /* Add IMS header */ |
798 | if (entry && entry->lastmod && request->method == METHOD_GET) { | |
1294c0fc | 799 | snprintf(ybuf, YBUF_SZ, "If-Modified-Since: %s", mkrfc1123(entry->lastmod)); |
88738790 | 800 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 1); |
6bf8443a | 801 | EBIT_SET(hdr_flags, HDR_IMS); |
802 | } | |
00c59270 | 803 | end = mime_headers_end(hdr_in); |
6bf8443a | 804 | for (t = hdr_in; t < end; t += strcspn(t, crlf), t += strspn(t, crlf)) { |
805 | hdr_len = t - hdr_in; | |
6bf8443a | 806 | l = strcspn(t, crlf) + 1; |
807 | if (l > 4096) | |
808 | l = 4096; | |
809 | xstrncpy(xbuf, t, l); | |
a3d5953d | 810 | debug(11, 5) ("httpBuildRequestHeader: %s\n", xbuf); |
6bf8443a | 811 | if (strncasecmp(xbuf, "Proxy-Connection:", 17) == 0) |
812 | continue; | |
88738790 | 813 | if (strncasecmp(xbuf, "Proxy-authorization:", 20) == 0) |
afe95a7e | 814 | /* If we're not going to do proxy auth, then it must be passed on */ |
79a15e0a | 815 | if (EBIT_TEST(request->flags, REQ_USED_PROXY_AUTH)) |
88738790 | 816 | continue; |
ab013258 | 817 | if (strncasecmp(xbuf, "Connection:", 11) == 0) { |
067bea91 | 818 | handleConnectionHeader(0, no_forward, &xbuf[11]); |
6bf8443a | 819 | continue; |
067bea91 | 820 | } |
66f7337b | 821 | if (strncasecmp(xbuf, "Host:", 5) == 0) { |
6bf8443a | 822 | EBIT_SET(hdr_flags, HDR_HOST); |
66f7337b | 823 | } else if (strncasecmp(xbuf, "Cache-Control:", 14) == 0) { |
6bf8443a | 824 | for (s = xbuf + 14; *s && isspace(*s); s++); |
825 | if (strncasecmp(s, "Max-age=", 8) == 0) | |
151a0b6d | 826 | EBIT_SET(cc_flags, CCC_MAXAGE); |
66f7337b | 827 | } else if (strncasecmp(xbuf, "Via:", 4) == 0) { |
6bf8443a | 828 | for (s = xbuf + 4; *s && isspace(*s); s++); |
a200bbd2 | 829 | if ((int)strlen(viabuf) + (int)strlen(s) < 4000) |
6bf8443a | 830 | strcat(viabuf, s); |
831 | strcat(viabuf, ", "); | |
832 | continue; | |
66f7337b | 833 | } else if (strncasecmp(xbuf, "X-Forwarded-For:", 16) == 0) { |
6bf8443a | 834 | for (s = xbuf + 16; *s && isspace(*s); s++); |
a200bbd2 | 835 | if ((int)strlen(fwdbuf) + (int)strlen(s) < 4000) |
6bf8443a | 836 | strcat(fwdbuf, s); |
837 | strcat(fwdbuf, ", "); | |
838 | continue; | |
66f7337b | 839 | } else if (strncasecmp(xbuf, "If-Modified-Since:", 18) == 0) { |
6bf8443a | 840 | if (EBIT_TEST(hdr_flags, HDR_IMS)) |
841 | continue; | |
b3b64e58 | 842 | } else if (strncasecmp(xbuf, "Max-Forwards:", 13) == 0) { |
843 | if (orig_request->method == METHOD_TRACE) { | |
844 | for (s = xbuf + 13; *s && isspace(*s); s++); | |
845 | n = atoi(s); | |
56878878 | 846 | snprintf(xbuf, 4096, "Max-Forwards: %d", n - 1); |
b3b64e58 | 847 | } |
66f7337b | 848 | } |
067bea91 | 849 | if (!handleConnectionHeader(1, no_forward, xbuf)) |
ab013258 | 850 | httpAppendRequestHeader(hdr_out, xbuf, &len, out_sz - 512, 1); |
88738790 | 851 | } |
852 | hdr_len = t - hdr_in; | |
853 | if (Config.fake_ua && strstr(hdr_out, "User-Agent") == NULL) { | |
1294c0fc | 854 | snprintf(ybuf, YBUF_SZ, "User-Agent: %s", Config.fake_ua); |
88738790 | 855 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 0); |
6bf8443a | 856 | } |
56878878 | 857 | /* Append Via: */ |
858 | /* snprintf would fail here too */ | |
1294c0fc | 859 | snprintf(ybuf, YBUF_SZ, "%3.1f %s", orig_request->http_ver, ThisCache); |
6bf8443a | 860 | strcat(viabuf, ybuf); |
88738790 | 861 | httpAppendRequestHeader(hdr_out, viabuf, &len, out_sz, 1); |
6bf8443a | 862 | /* Append to X-Forwarded-For: */ |
a08307eb | 863 | strcat(fwdbuf, cfd < 0 ? "unknown" : fd_table[cfd].ipaddr); |
88738790 | 864 | httpAppendRequestHeader(hdr_out, fwdbuf, &len, out_sz, 1); |
6bf8443a | 865 | if (!EBIT_TEST(hdr_flags, HDR_HOST)) { |
1294c0fc | 866 | snprintf(ybuf, YBUF_SZ, "Host: %s", orig_request->host); |
88738790 | 867 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 1); |
6bf8443a | 868 | } |
151a0b6d | 869 | if (!EBIT_TEST(cc_flags, CCC_MAXAGE)) { |
9fb13bb6 | 870 | url = entry ? storeUrl(entry) : urlCanonical(orig_request, NULL); |
1294c0fc | 871 | snprintf(ybuf, YBUF_SZ, "Cache-control: Max-age=%d", (int) getMaxAge(url)); |
88738790 | 872 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 1); |
79d39a72 | 873 | if (request->urlpath[0]) |
365e5b34 | 874 | assert(strstr(url, request->urlpath)); |
6bf8443a | 875 | } |
603a02fd | 876 | /* maybe append Connection: Keep-Alive */ |
79a15e0a | 877 | if (EBIT_TEST(flags, HTTP_KEEPALIVE)) { |
878 | if (EBIT_TEST(flags, HTTP_PROXYING)) { | |
1294c0fc | 879 | snprintf(ybuf, YBUF_SZ, "Proxy-Connection: Keep-Alive"); |
603a02fd | 880 | } else { |
1294c0fc | 881 | snprintf(ybuf, YBUF_SZ, "Connection: Keep-Alive"); |
603a02fd | 882 | } |
883 | httpAppendRequestHeader(hdr_out, ybuf, &len, out_sz, 1); | |
884 | } | |
88738790 | 885 | httpAppendRequestHeader(hdr_out, null_string, &len, out_sz, 1); |
3f6c0fb2 | 886 | memFree(MEM_4K_BUF, xbuf); |
887 | memFree(MEM_4K_BUF, viabuf); | |
888 | memFree(MEM_4K_BUF, fwdbuf); | |
6bf8443a | 889 | if (in_len) |
890 | *in_len = hdr_len; | |
9d9d144b | 891 | if ((l = strlen(hdr_out)) != len) { |
892 | debug_trap("httpBuildRequestHeader: size mismatch"); | |
893 | len = l; | |
894 | } | |
a3d5953d | 895 | debug(11, 3) ("httpBuildRequestHeader: OUTPUT:\n%s\n", hdr_out); |
6bf8443a | 896 | return len; |
897 | } | |
898 | ||
090089c4 | 899 | /* This will be called when connect completes. Write request. */ |
b8d8561b | 900 | static void |
b177367b | 901 | httpSendRequest(int fd, void *data) |
090089c4 | 902 | { |
b177367b | 903 | HttpStateData *httpState = data; |
090089c4 | 904 | char *buf = NULL; |
090089c4 | 905 | int len = 0; |
906 | int buflen; | |
30a4f2a8 | 907 | request_t *req = httpState->request; |
9864ee44 | 908 | int buftype = 0; |
620da955 | 909 | StoreEntry *entry = httpState->entry; |
2a26c096 | 910 | int cfd; |
1294c0fc | 911 | peer *p = httpState->peer; |
090089c4 | 912 | |
a3d5953d | 913 | debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", fd, httpState); |
6bf8443a | 914 | buflen = strlen(req->urlpath); |
62dec5a7 | 915 | if (req->headers) |
916 | buflen += req->headers_sz + 1; | |
090089c4 | 917 | buflen += 512; /* lots of extra */ |
918 | ||
78657218 | 919 | if ((req->method == METHOD_POST || req->method == METHOD_PUT)) { |
920 | debug_trap("httpSendRequest: should not be handling POST/PUT request"); | |
921 | return; | |
090089c4 | 922 | } |
30a4f2a8 | 923 | if (buflen < DISK_PAGE_SIZE) { |
3f6c0fb2 | 924 | buf = memAllocate(MEM_8K_BUF, 1); |
9864ee44 | 925 | buftype = BUF_TYPE_8K; |
6bf8443a | 926 | buflen = DISK_PAGE_SIZE; |
30a4f2a8 | 927 | } else { |
9864ee44 | 928 | buf = xcalloc(buflen, 1); |
929 | buftype = BUF_TYPE_MALLOC; | |
090089c4 | 930 | } |
2a26c096 | 931 | if (!opt_forwarded_for) |
6bf8443a | 932 | cfd = -1; |
2a26c096 | 933 | else if (entry->mem_obj == NULL) |
6bf8443a | 934 | cfd = -1; |
2a26c096 | 935 | else |
382d851a | 936 | cfd = entry->mem_obj->fd; |
1294c0fc | 937 | if (p != NULL) |
79a15e0a | 938 | EBIT_SET(httpState->flags, HTTP_PROXYING); |
1294c0fc | 939 | if (req->method == METHOD_GET) { |
9a47da71 | 940 | if (p == NULL) |
941 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); | |
942 | else if (p->stats.n_keepalives_sent < 10) | |
943 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); | |
944 | else if ((double) p->stats.n_keepalives_recv / (double) p->stats.n_keepalives_sent > 0.50) | |
79a15e0a | 945 | EBIT_SET(httpState->flags, HTTP_KEEPALIVE); |
1294c0fc | 946 | } |
6bf8443a | 947 | len = httpBuildRequestHeader(req, |
79a15e0a | 948 | httpState->orig_request, |
6bf8443a | 949 | entry, |
6bf8443a | 950 | NULL, |
951 | buf, | |
952 | buflen, | |
603a02fd | 953 | cfd, |
954 | httpState->flags); | |
a3d5953d | 955 | debug(11, 6) ("httpSendRequest: FD %d:\n%s\n", fd, buf); |
30a4f2a8 | 956 | comm_write(fd, |
14e59844 | 957 | buf, |
958 | len, | |
14e59844 | 959 | httpSendComplete, |
9864ee44 | 960 | httpState, |
3f6c0fb2 | 961 | buftype == BUF_TYPE_8K ? memFree8K : xfree); |
52ecbd09 | 962 | #ifdef BREAKS_PCONN_RESTART |
20cc1450 | 963 | requestUnlink(httpState->orig_request); |
964 | httpState->orig_request = NULL; | |
52ecbd09 | 965 | #endif |
090089c4 | 966 | } |
967 | ||
603a02fd | 968 | static int |
b716a8ad | 969 | httpSocketOpen(StoreEntry * entry, request_t * request) |
090089c4 | 970 | { |
9e4ad609 | 971 | int fd; |
9b312a19 | 972 | ErrorState *err; |
9e4ad609 | 973 | fd = comm_open(SOCK_STREAM, |
16b204c4 | 974 | 0, |
975 | Config.Addrs.tcp_outgoing, | |
976 | 0, | |
977 | COMM_NONBLOCKING, | |
9fb13bb6 | 978 | storeUrl(entry)); |
603a02fd | 979 | if (fd < 0) { |
79a15e0a | 980 | debug(50, 4) ("httpSocketOpen: %s\n", xstrerror()); |
fe40a877 | 981 | err = errorCon(ERR_SOCKET_FAILURE, HTTP_INTERNAL_SERVER_ERROR); |
c45ed9ad | 982 | err->xerrno = errno; |
79a15e0a | 983 | err->request = requestLink(request); |
9b312a19 | 984 | errorAppendEntry(entry, err); |
090089c4 | 985 | } |
603a02fd | 986 | return fd; |
987 | } | |
988 | ||
989 | static HttpStateData * | |
990 | httpBuildState(int fd, StoreEntry * entry, request_t * orig_request, peer * e) | |
991 | { | |
992 | HttpStateData *httpState = xcalloc(1, sizeof(HttpStateData)); | |
993 | request_t *request; | |
770f051d | 994 | storeLockObject(entry); |
3f6c0fb2 | 995 | cbdataAdd(httpState, MEM_NONE); |
0a0bf5db | 996 | httpState->entry = entry; |
9e4ad609 | 997 | httpState->fd = fd; |
603a02fd | 998 | if (e) { |
3f6c0fb2 | 999 | request = memAllocate(MEM_REQUEST_T, 1); |
603a02fd | 1000 | request->method = orig_request->method; |
1001 | xstrncpy(request->host, e->host, SQUIDHOSTNAMELEN); | |
1002 | request->port = e->http_port; | |
9fb13bb6 | 1003 | xstrncpy(request->urlpath, storeUrl(entry), MAX_URL); |
603a02fd | 1004 | httpState->request = requestLink(request); |
1294c0fc | 1005 | httpState->peer = e; |
603a02fd | 1006 | httpState->orig_request = requestLink(orig_request); |
79a15e0a | 1007 | EBIT_SET(request->flags, REQ_PROXYING); |
603a02fd | 1008 | } else { |
1009 | httpState->request = requestLink(orig_request); | |
79a15e0a | 1010 | httpState->orig_request = requestLink(orig_request); |
603a02fd | 1011 | } |
0d4d4170 | 1012 | /* register the handler to free HTTP state data when the FD closes */ |
603a02fd | 1013 | comm_add_close_handler(httpState->fd, httpStateFree, httpState); |
e102ebda | 1014 | storeRegisterAbort(entry, httpAbort, httpState); |
603a02fd | 1015 | return httpState; |
1016 | } | |
1017 | ||
1018 | void | |
1019 | httpStart(request_t * request, StoreEntry * entry, peer * e) | |
1020 | { | |
1021 | HttpStateData *httpState; | |
1022 | int fd; | |
1023 | debug(11, 3) ("httpStart: \"%s %s\"\n", | |
9fb13bb6 | 1024 | RequestMethodStr[request->method], storeUrl(entry)); |
603a02fd | 1025 | if (e) { |
1026 | if (e->options & NEIGHBOR_PROXY_ONLY) | |
603a02fd | 1027 | storeReleaseRequest(entry); |
603a02fd | 1028 | if ((fd = pconnPop(e->host, e->http_port)) >= 0) { |
51fdcbd5 | 1029 | debug(11, 3) ("httpStart: reusing pconn FD %d\n", fd); |
603a02fd | 1030 | httpState = httpBuildState(fd, entry, request, e); |
b716a8ad | 1031 | commSetTimeout(httpState->fd, |
1032 | Config.Timeout.connect, | |
1033 | httpTimeout, | |
1034 | httpState); | |
603a02fd | 1035 | httpConnectDone(fd, COMM_OK, httpState); |
1036 | return; | |
1037 | } | |
1038 | } else { | |
1039 | if ((fd = pconnPop(request->host, request->port)) >= 0) { | |
51fdcbd5 | 1040 | debug(11, 3) ("httpStart: reusing pconn FD %d\n", fd); |
603a02fd | 1041 | httpState = httpBuildState(fd, entry, request, e); |
b716a8ad | 1042 | commSetTimeout(httpState->fd, |
1043 | Config.Timeout.connect, | |
1044 | httpTimeout, | |
1045 | httpState); | |
603a02fd | 1046 | httpConnectDone(fd, COMM_OK, httpState); |
1047 | return; | |
1048 | } | |
1049 | } | |
79a15e0a | 1050 | if ((fd = httpSocketOpen(entry, request)) < 0) |
603a02fd | 1051 | return; |
1052 | httpState = httpBuildState(fd, entry, request, e); | |
603a02fd | 1053 | commSetTimeout(httpState->fd, |
1054 | Config.Timeout.connect, | |
1055 | httpTimeout, | |
cd1fb0eb | 1056 | httpState); |
edeb28fd | 1057 | commConnectStart(httpState->fd, |
603a02fd | 1058 | httpState->request->host, |
1059 | httpState->request->port, | |
e924600d | 1060 | httpConnectDone, |
1061 | httpState); | |
e5f6c5c2 | 1062 | } |
1063 | ||
b716a8ad | 1064 | static void |
1065 | httpRestart(HttpStateData * httpState) | |
1066 | { | |
1067 | /* restart a botched request from a persistent connection */ | |
9fb13bb6 | 1068 | debug(11, 2) ("Retrying HTTP request for %s\n", storeUrl(httpState->entry)); |
b716a8ad | 1069 | if (httpState->fd >= 0) { |
1070 | comm_remove_close_handler(httpState->fd, httpStateFree, httpState); | |
1071 | comm_close(httpState->fd); | |
1072 | httpState->fd = -1; | |
1073 | } | |
79a15e0a | 1074 | httpState->fd = httpSocketOpen(httpState->entry, httpState->orig_request); |
b716a8ad | 1075 | if (httpState->fd < 0) |
1076 | return; | |
1077 | comm_add_close_handler(httpState->fd, httpStateFree, httpState); | |
1078 | commSetTimeout(httpState->fd, | |
1079 | Config.Timeout.connect, | |
1080 | httpTimeout, | |
1081 | httpState); | |
1082 | commConnectStart(httpState->fd, | |
1083 | httpState->request->host, | |
1084 | httpState->request->port, | |
1085 | httpConnectDone, | |
1086 | httpState); | |
1087 | } | |
1088 | ||
e5f6c5c2 | 1089 | static void |
1090 | httpConnectDone(int fd, int status, void *data) | |
1091 | { | |
1092 | HttpStateData *httpState = data; | |
1093 | request_t *request = httpState->request; | |
1094 | StoreEntry *entry = httpState->entry; | |
9b312a19 | 1095 | ErrorState *err; |
edeb28fd | 1096 | if (status == COMM_ERR_DNS) { |
a3d5953d | 1097 | debug(11, 4) ("httpConnectDone: Unknown host: %s\n", request->host); |
fe40a877 | 1098 | err = errorCon(ERR_DNS_FAIL, HTTP_SERVICE_UNAVAILABLE); |
9b312a19 | 1099 | err->dnsserver_msg = xstrdup(dns_error_message); |
79a15e0a | 1100 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 1101 | errorAppendEntry(entry, err); |
edeb28fd | 1102 | comm_close(fd); |
1103 | } else if (status != COMM_OK) { | |
fe40a877 | 1104 | err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE); |
c45ed9ad | 1105 | err->xerrno = errno; |
9b312a19 | 1106 | err->host = xstrdup(request->host); |
1107 | err->port = request->port; | |
79a15e0a | 1108 | err->request = requestLink(httpState->orig_request); |
9b312a19 | 1109 | errorAppendEntry(entry, err); |
1294c0fc | 1110 | if (httpState->peer) |
1111 | peerCheckConnectStart(httpState->peer); | |
e5f6c5c2 | 1112 | comm_close(fd); |
1113 | } else { | |
9fb13bb6 | 1114 | fd_note(fd, storeUrl(entry)); |
b716a8ad | 1115 | fd_table[fd].uses++; |
bfcaf585 | 1116 | commSetSelect(fd, COMM_SELECT_WRITE, httpSendRequest, httpState, 0); |
090089c4 | 1117 | } |
090089c4 | 1118 | } |
1119 | ||
b8d8561b | 1120 | void |
1121 | httpReplyHeaderStats(StoreEntry * entry) | |
30a4f2a8 | 1122 | { |
6fb52f6c | 1123 | http_server_cc_t i; |
151a0b6d | 1124 | http_hdr_misc_t j; |
30a4f2a8 | 1125 | storeAppendPrintf(entry, open_bracket); |
6fb52f6c | 1126 | storeAppendPrintf(entry, "{HTTP Reply Headers:}\n"); |
1127 | storeAppendPrintf(entry, "{ Headers parsed: %d}\n", | |
30a4f2a8 | 1128 | ReplyHeaderStats.parsed); |
151a0b6d | 1129 | for (j = HDR_AGE; j < HDR_MISC_END; j++) |
1130 | storeAppendPrintf(entry, "{%21.21s: %d}\n", | |
1131 | HttpHdrMiscStr[j], | |
1132 | ReplyHeaderStats.misc[j]); | |
4db43fab | 1133 | for (i = SCC_PUBLIC; i < SCC_ENUM_END; i++) |
884011a1 | 1134 | storeAppendPrintf(entry, "{Cache-Control %s: %d}\n", |
6fb52f6c | 1135 | HttpServerCCStr[i], |
1136 | ReplyHeaderStats.cc[i]); | |
30a4f2a8 | 1137 | storeAppendPrintf(entry, close_bracket); |
1138 | } | |
bfcaf585 | 1139 | |
1140 | static void | |
1141 | httpAbort(void *data) | |
1142 | { | |
1143 | HttpStateData *httpState = data; | |
9fb13bb6 | 1144 | debug(11, 2) ("httpAbort: %s\n", storeUrl(httpState->entry)); |
bfcaf585 | 1145 | comm_close(httpState->fd); |
1146 | } | |
9b312a19 | 1147 | |
1148 | static char * | |
1149 | httpStatusString(int status) | |
1150 | { | |
1151 | char *p = NULL; | |
1152 | switch (status) { | |
1153 | case 100: | |
1154 | p = "Continue"; | |
1155 | break; | |
1156 | case 101: | |
1157 | p = "Switching Protocols"; | |
1158 | break; | |
1159 | case 200: | |
1160 | p = "OK"; | |
1161 | break; | |
1162 | case 201: | |
1163 | p = "Created"; | |
1164 | break; | |
1165 | case 202: | |
1166 | p = "Accepted"; | |
1167 | break; | |
1168 | case 203: | |
1169 | p = "Non-Authoritative Information"; | |
1170 | break; | |
1171 | case 204: | |
1172 | p = "No Content"; | |
1173 | break; | |
1174 | case 205: | |
1175 | p = "Reset Content"; | |
1176 | break; | |
1177 | case 206: | |
1178 | p = "Partial Content"; | |
1179 | break; | |
1180 | case 300: | |
1181 | p = "Multiple Choices"; | |
1182 | break; | |
1183 | case 301: | |
1184 | p = "Moved Permanently"; | |
1185 | break; | |
1186 | case 302: | |
1187 | p = "Moved Temporarily"; | |
1188 | break; | |
1189 | case 303: | |
1190 | p = "See Other"; | |
1191 | break; | |
1192 | case 304: | |
1193 | p = "Not Modified"; | |
1194 | break; | |
1195 | case 305: | |
1196 | p = "Use Proxy"; | |
1197 | break; | |
1198 | case 400: | |
1199 | p = "Bad Request"; | |
1200 | break; | |
1201 | case 401: | |
1202 | p = "Unauthorized"; | |
1203 | break; | |
1204 | case 402: | |
1205 | p = "Payment Required"; | |
1206 | break; | |
1207 | case 403: | |
1208 | p = "Forbidden"; | |
1209 | break; | |
1210 | case 404: | |
1211 | p = "Not Found"; | |
1212 | break; | |
1213 | case 405: | |
1214 | p = "Method Not Allowed"; | |
1215 | break; | |
1216 | case 406: | |
1217 | p = "Not Acceptable"; | |
1218 | break; | |
1219 | case 407: | |
1220 | p = "Proxy Authentication Required"; | |
1221 | break; | |
1222 | case 408: | |
1223 | p = "Request Time-out"; | |
1224 | break; | |
1225 | case 409: | |
1226 | p = "Conflict"; | |
1227 | break; | |
1228 | case 410: | |
1229 | p = "Gone"; | |
1230 | break; | |
1231 | case 411: | |
1232 | p = "Length Required"; | |
1233 | break; | |
1234 | case 412: | |
1235 | p = "Precondition Failed"; | |
1236 | break; | |
1237 | case 413: | |
1238 | p = "Request Entity Too Large"; | |
1239 | break; | |
1240 | case 414: | |
1241 | p = "Request-URI Too Large"; | |
1242 | break; | |
1243 | case 415: | |
1244 | p = "Unsupported Media Type"; | |
1245 | break; | |
1246 | case 500: | |
1247 | p = "Internal Server Error"; | |
1248 | break; | |
1249 | case 501: | |
1250 | p = "Not Implemented"; | |
1251 | break; | |
1252 | case 502: | |
1253 | p = "Bad Gateway"; | |
1254 | break; | |
1255 | case 503: | |
1256 | p = "Service Unavailable"; | |
1257 | break; | |
1258 | case 504: | |
1259 | p = "Gateway Time-out"; | |
1260 | break; | |
1261 | case 505: | |
1262 | p = "HTTP Version not supported"; | |
1263 | break; | |
1264 | default: | |
1265 | p = "Unknown"; | |
1266 | debug(11, 0) ("Unknown HTTP status code: %d\n", status); | |
1267 | break; | |
1268 | } | |
1269 | return p; | |
1270 | } | |
1271 | ||
1272 | char * | |
1273 | httpReplyHeader(double ver, | |
1274 | http_status status, | |
1275 | char *ctype, | |
1276 | int clen, | |
1277 | time_t lmt, | |
1278 | time_t expires) | |
1279 | { | |
1280 | LOCAL_ARRAY(char, buf, HTTP_REPLY_BUF_SZ); | |
9b312a19 | 1281 | int l = 0; |
1282 | int s = HTTP_REPLY_BUF_SZ; | |
fc5d6f7f | 1283 | l += snprintf(buf + l, s - l, "HTTP/%3.1f %d %s\r\n", |
1284 | ver, | |
9b312a19 | 1285 | (int) status, |
1286 | httpStatusString(status)); | |
1287 | l += snprintf(buf + l, s - l, "Server: Squid/%s\r\n", SQUID_VERSION); | |
1288 | l += snprintf(buf + l, s - l, "Date: %s\r\n", mkrfc1123(squid_curtime)); | |
1289 | if (expires >= 0) | |
1290 | l += snprintf(buf + l, s - l, "Expires: %s\r\n", mkrfc1123(expires)); | |
1291 | if (lmt) | |
1292 | l += snprintf(buf + l, s - l, "Last-Modified: %s\r\n", mkrfc1123(lmt)); | |
1293 | if (clen > 0) | |
1294 | l += snprintf(buf + l, s - l, "Content-Length: %d\r\n", clen); | |
1295 | if (ctype) | |
1296 | l += snprintf(buf + l, s - l, "Content-Type: %s\r\n", ctype); | |
1297 | return buf; | |
1298 | } |