]>
Commit | Line | Data |
---|---|---|
30a4f2a8 | 1 | /* |
78657218 | 2 | * $Id: http.cc,v 1.111 1996/11/22 08:37:00 wessels Exp $ |
30a4f2a8 | 3 | * |
4 | * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) | |
5 | * AUTHOR: Harvest Derived | |
6 | * | |
42c04c16 | 7 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
30a4f2a8 | 8 | * -------------------------------------------------------- |
9 | * | |
10 | * Squid is the result of efforts by numerous individuals from the | |
11 | * Internet community. Development is led by Duane Wessels of the | |
12 | * National Laboratory for Applied Network Research and funded by | |
13 | * the National Science Foundation. | |
14 | * | |
15 | * This program is free software; you can redistribute it and/or modify | |
16 | * it under the terms of the GNU General Public License as published by | |
17 | * the Free Software Foundation; either version 2 of the License, or | |
18 | * (at your option) any later version. | |
19 | * | |
20 | * This program is distributed in the hope that it will be useful, | |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
24 | * | |
25 | * You should have received a copy of the GNU General Public License | |
26 | * along with this program; if not, write to the Free Software | |
27 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
28 | * | |
29 | */ | |
019dd986 | 30 | |
31 | /* | |
30a4f2a8 | 32 | * Copyright (c) 1994, 1995. All rights reserved. |
33 | * | |
34 | * The Harvest software was developed by the Internet Research Task | |
35 | * Force Research Group on Resource Discovery (IRTF-RD): | |
36 | * | |
37 | * Mic Bowman of Transarc Corporation. | |
38 | * Peter Danzig of the University of Southern California. | |
39 | * Darren R. Hardy of the University of Colorado at Boulder. | |
40 | * Udi Manber of the University of Arizona. | |
41 | * Michael F. Schwartz of the University of Colorado at Boulder. | |
42 | * Duane Wessels of the University of Colorado at Boulder. | |
43 | * | |
44 | * This copyright notice applies to software in the Harvest | |
45 | * ``src/'' directory only. Users should consult the individual | |
46 | * copyright notices in the ``components/'' subdirectories for | |
47 | * copyright information about other software bundled with the | |
48 | * Harvest source code distribution. | |
49 | * | |
50 | * TERMS OF USE | |
51 | * | |
52 | * The Harvest software may be used and re-distributed without | |
53 | * charge, provided that the software origin and research team are | |
54 | * cited in any use of the system. Most commonly this is | |
55 | * accomplished by including a link to the Harvest Home Page | |
56 | * (http://harvest.cs.colorado.edu/) from the query page of any | |
57 | * Broker you deploy, as well as in the query result pages. These | |
58 | * links are generated automatically by the standard Broker | |
59 | * software distribution. | |
60 | * | |
61 | * The Harvest software is provided ``as is'', without express or | |
62 | * implied warranty, and with no support nor obligation to assist | |
63 | * in its use, correction, modification or enhancement. We assume | |
64 | * no liability with respect to the infringement of copyrights, | |
65 | * trade secrets, or any patents, and are not responsible for | |
66 | * consequential damages. Proper use of the Harvest software is | |
67 | * entirely the responsibility of the user. | |
68 | * | |
69 | * DERIVATIVE WORKS | |
70 | * | |
71 | * Users may make derivative works from the Harvest software, subject | |
72 | * to the following constraints: | |
73 | * | |
74 | * - You must include the above copyright notice and these | |
75 | * accompanying paragraphs in all forms of derivative works, | |
76 | * and any documentation and other materials related to such | |
77 | * distribution and use acknowledge that the software was | |
78 | * developed at the above institutions. | |
79 | * | |
80 | * - You must notify IRTF-RD regarding your distribution of | |
81 | * the derivative work. | |
82 | * | |
83 | * - You must clearly notify users that your are distributing | |
84 | * a modified version and not the original Harvest software. | |
85 | * | |
86 | * - Any derivative product is also subject to these copyright | |
87 | * and use restrictions. | |
88 | * | |
89 | * Note that the Harvest software is NOT in the public domain. We | |
90 | * retain copyright, as specified above. | |
91 | * | |
92 | * HISTORY OF FREE SOFTWARE STATUS | |
93 | * | |
94 | * Originally we required sites to license the software in cases | |
95 | * where they were going to build commercial products/services | |
96 | * around Harvest. In June 1995 we changed this policy. We now | |
97 | * allow people to use the core Harvest software (the code found in | |
98 | * the Harvest ``src/'' directory) for free. We made this change | |
99 | * in the interest of encouraging the widest possible deployment of | |
100 | * the technology. The Harvest software is really a reference | |
101 | * implementation of a set of protocols and formats, some of which | |
102 | * we intend to standardize. We encourage commercial | |
103 | * re-implementations of code complying to this set of standards. | |
019dd986 | 104 | */ |
44a47c6e | 105 | |
106 | #include "squid.h" | |
090089c4 | 107 | |
234967c9 | 108 | #define HTTP_DELETE_GAP (1<<18) |
090089c4 | 109 | |
4db43fab | 110 | static const char *const w_space = " \t\n\r"; |
111 | ||
6fb52f6c | 112 | typedef enum { |
113 | SCC_PUBLIC, | |
114 | SCC_PRIVATE, | |
115 | SCC_NOCACHE, | |
116 | SCC_NOSTORE, | |
117 | SCC_NOTRANSFORM, | |
118 | SCC_MUSTREVALIDATE, | |
119 | SCC_PROXYREVALIDATE, | |
120 | SCC_MAXAGE, | |
121 | SCC_ENUM_END | |
122 | } http_server_cc_t; | |
123 | ||
124 | typedef enum { | |
125 | CCC_NOCACHE, | |
126 | CCC_NOSTORE, | |
127 | CCC_MAXAGE, | |
128 | CCC_MAXSTALE, | |
129 | CCC_MINFRESH, | |
130 | CCC_ONLYIFCACHED, | |
131 | CCC_ENUM_END | |
132 | } http_client_cc_t; | |
133 | ||
134 | char *HttpServerCCStr[] = | |
135 | { | |
136 | "public", | |
137 | "private", | |
138 | "no-cache", | |
139 | "no-store", | |
140 | "no-transform", | |
141 | "must-revalidate", | |
142 | "proxy-revalidate", | |
143 | "max-age", | |
144 | "NONE" | |
145 | }; | |
146 | ||
24382924 | 147 | static struct { |
30a4f2a8 | 148 | int parsed; |
149 | int date; | |
150 | int lm; | |
151 | int exp; | |
152 | int clen; | |
153 | int ctype; | |
6fb52f6c | 154 | int cc[SCC_ENUM_END]; |
30a4f2a8 | 155 | } ReplyHeaderStats; |
090089c4 | 156 | |
b177367b | 157 | static void httpStateFree _PARAMS((int fd, void *)); |
158 | static void httpReadReplyTimeout _PARAMS((int fd, void *)); | |
159 | static void httpLifetimeExpire _PARAMS((int fd, void *)); | |
67508012 | 160 | static void httpMakePublic _PARAMS((StoreEntry *)); |
161 | static void httpMakePrivate _PARAMS((StoreEntry *)); | |
162 | static void httpCacheNegatively _PARAMS((StoreEntry *)); | |
b177367b | 163 | static void httpReadReply _PARAMS((int fd, void *)); |
67508012 | 164 | static void httpSendComplete _PARAMS((int fd, char *, int, int, void *)); |
b177367b | 165 | static void httpSendRequest _PARAMS((int fd, void *)); |
0ee4272b | 166 | static void httpConnect _PARAMS((int fd, const ipcache_addrs *, void *)); |
e5f6c5c2 | 167 | static void httpConnectDone _PARAMS((int fd, int status, void *data)); |
b8d8561b | 168 | |
b177367b | 169 | static void |
170 | httpStateFree(int fd, void *data) | |
f5558c95 | 171 | { |
b177367b | 172 | HttpStateData *httpState = data; |
0d4d4170 | 173 | if (httpState == NULL) |
b177367b | 174 | return; |
30a4f2a8 | 175 | storeUnlockObject(httpState->entry); |
0d4d4170 | 176 | if (httpState->reply_hdr) { |
177 | put_free_8k_page(httpState->reply_hdr); | |
178 | httpState->reply_hdr = NULL; | |
179 | } | |
30a4f2a8 | 180 | requestUnlink(httpState->request); |
0d4d4170 | 181 | xfree(httpState); |
f5558c95 | 182 | } |
183 | ||
b8d8561b | 184 | int |
0ee4272b | 185 | httpCachable(const char *url, int method) |
090089c4 | 186 | { |
090089c4 | 187 | /* GET and HEAD are cachable. Others are not. */ |
6eb42cae | 188 | if (method != METHOD_GET && method != METHOD_HEAD) |
090089c4 | 189 | return 0; |
090089c4 | 190 | /* else cachable */ |
191 | return 1; | |
192 | } | |
193 | ||
194 | /* This will be called when timeout on read. */ | |
b8d8561b | 195 | static void |
b177367b | 196 | httpReadReplyTimeout(int fd, void *data) |
090089c4 | 197 | { |
b177367b | 198 | HttpStateData *httpState = data; |
090089c4 | 199 | StoreEntry *entry = NULL; |
30a4f2a8 | 200 | entry = httpState->entry; |
593c9a75 | 201 | debug(11, 4, "httpReadReplyTimeout: FD %d: '%s'\n", fd, entry->url); |
b8de7ebe | 202 | squid_error_entry(entry, ERR_READ_TIMEOUT, NULL); |
b177367b | 203 | commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); |
0d4d4170 | 204 | comm_close(fd); |
090089c4 | 205 | } |
206 | ||
207 | /* This will be called when socket lifetime is expired. */ | |
b8d8561b | 208 | static void |
b177367b | 209 | httpLifetimeExpire(int fd, void *data) |
090089c4 | 210 | { |
b177367b | 211 | HttpStateData *httpState = data; |
593c9a75 | 212 | StoreEntry *entry = httpState->entry; |
213 | debug(11, 4, "httpLifeTimeExpire: FD %d: '%s'\n", fd, entry->url); | |
ce49f524 | 214 | squid_error_entry(entry, ERR_LIFETIME_EXP, NULL); |
b177367b | 215 | commSetSelect(fd, COMM_SELECT_READ | COMM_SELECT_WRITE, NULL, NULL, 0); |
0d4d4170 | 216 | comm_close(fd); |
090089c4 | 217 | } |
218 | ||
30a4f2a8 | 219 | /* This object can be cached for a long time */ |
b8d8561b | 220 | static void |
221 | httpMakePublic(StoreEntry * entry) | |
30a4f2a8 | 222 | { |
1c481e00 | 223 | if (BIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 224 | storeSetPublicKey(entry); |
225 | } | |
226 | ||
227 | /* This object should never be cached at all */ | |
b8d8561b | 228 | static void |
229 | httpMakePrivate(StoreEntry * entry) | |
30a4f2a8 | 230 | { |
30a4f2a8 | 231 | storeExpireNow(entry); |
1c481e00 | 232 | BIT_RESET(entry->flag, ENTRY_CACHABLE); |
30a4f2a8 | 233 | storeReleaseRequest(entry); /* delete object when not used */ |
234 | } | |
235 | ||
236 | /* This object may be negatively cached */ | |
b8d8561b | 237 | static void |
238 | httpCacheNegatively(StoreEntry * entry) | |
30a4f2a8 | 239 | { |
79b5cc5f | 240 | storeNegativeCache(entry); |
1c481e00 | 241 | if (BIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 242 | storeSetPublicKey(entry); |
30a4f2a8 | 243 | } |
244 | ||
245 | ||
246 | /* Build a reply structure from HTTP reply headers */ | |
b8d8561b | 247 | void |
48f44632 | 248 | httpParseReplyHeaders(const char *buf, struct _http_reply *reply) |
30a4f2a8 | 249 | { |
250 | char *headers = NULL; | |
251 | char *t = NULL; | |
252 | char *s = NULL; | |
6fb52f6c | 253 | int delta; |
30a4f2a8 | 254 | |
255 | ReplyHeaderStats.parsed++; | |
256 | headers = xstrdup(buf); | |
257 | t = strtok(headers, "\n"); | |
258 | while (t) { | |
259 | s = t + strlen(t); | |
260 | while (*s == '\r') | |
261 | *s-- = '\0'; | |
262 | if (!strncasecmp(t, "HTTP", 4)) { | |
263 | sscanf(t + 1, "%lf", &reply->version); | |
264 | if ((t = strchr(t, ' '))) { | |
265 | t++; | |
266 | reply->code = atoi(t); | |
267 | } | |
268 | } else if (!strncasecmp(t, "Content-type:", 13)) { | |
269 | if ((t = strchr(t, ' '))) { | |
270 | t++; | |
f2052513 | 271 | xstrncpy(reply->content_type, t, HTTP_REPLY_FIELD_SZ); |
30a4f2a8 | 272 | ReplyHeaderStats.ctype++; |
273 | } | |
274 | } else if (!strncasecmp(t, "Content-length:", 15)) { | |
275 | if ((t = strchr(t, ' '))) { | |
276 | t++; | |
277 | reply->content_length = atoi(t); | |
278 | ReplyHeaderStats.clen++; | |
279 | } | |
280 | } else if (!strncasecmp(t, "Date:", 5)) { | |
281 | if ((t = strchr(t, ' '))) { | |
282 | t++; | |
f2052513 | 283 | xstrncpy(reply->date, t, HTTP_REPLY_FIELD_SZ); |
30a4f2a8 | 284 | ReplyHeaderStats.date++; |
285 | } | |
286 | } else if (!strncasecmp(t, "Expires:", 8)) { | |
287 | if ((t = strchr(t, ' '))) { | |
288 | t++; | |
f2052513 | 289 | xstrncpy(reply->expires, t, HTTP_REPLY_FIELD_SZ); |
30a4f2a8 | 290 | ReplyHeaderStats.exp++; |
291 | } | |
292 | } else if (!strncasecmp(t, "Last-Modified:", 14)) { | |
293 | if ((t = strchr(t, ' '))) { | |
294 | t++; | |
f2052513 | 295 | xstrncpy(reply->last_modified, t, HTTP_REPLY_FIELD_SZ); |
30a4f2a8 | 296 | ReplyHeaderStats.lm++; |
297 | } | |
caebbe00 | 298 | } else if (!strncasecmp(t, "Cache-Control:", 14)) { |
4db43fab | 299 | t += 14; |
300 | while (*t == ' ' || *t == '\t') | |
301 | t++; | |
302 | if (!strncasecmp(t, "public", 6)) { | |
303 | EBIT_SET(reply->cache_control, SCC_PUBLIC); | |
304 | ReplyHeaderStats.cc[SCC_PUBLIC]++; | |
305 | } else if (!strncasecmp(t, "private", 7)) { | |
306 | EBIT_SET(reply->cache_control, SCC_PRIVATE); | |
307 | ReplyHeaderStats.cc[SCC_PRIVATE]++; | |
308 | } else if (!strncasecmp(t, "no-cache", 8)) { | |
309 | EBIT_SET(reply->cache_control, SCC_NOCACHE); | |
310 | ReplyHeaderStats.cc[SCC_NOCACHE]++; | |
311 | } else if (!strncasecmp(t, "max-age", 7)) { | |
312 | if ((t = strchr(t, '='))) { | |
313 | delta = atoi(++t); | |
314 | EBIT_SET(reply->cache_control, SCC_MAXAGE); | |
315 | ReplyHeaderStats.cc[SCC_MAXAGE]++; | |
316 | strcpy(reply->expires, mkrfc1123(squid_curtime + delta)); | |
5df61230 | 317 | } |
caebbe00 | 318 | } |
30a4f2a8 | 319 | } |
320 | t = strtok(NULL, "\n"); | |
321 | } | |
1c481e00 | 322 | #if LOG_TIMESTAMPS |
323 | fprintf(timestamp_log, "T %9d D %9d L %9d E %9d\n", | |
324 | squid_curtime, | |
cf49df4b | 325 | parse_rfc1123(reply->date), |
326 | parse_rfc1123(reply->last_modified), | |
327 | parse_rfc1123(reply->expires)); | |
1c481e00 | 328 | #endif /* LOG_TIMESTAMPS */ |
30a4f2a8 | 329 | safe_free(headers); |
330 | } | |
331 | ||
090089c4 | 332 | |
b8d8561b | 333 | void |
0ee4272b | 334 | httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) |
f5558c95 | 335 | { |
336 | char *t = NULL; | |
30a4f2a8 | 337 | StoreEntry *entry = httpState->entry; |
d3fb4dea | 338 | int room; |
339 | int hdr_len; | |
2285407f | 340 | struct _http_reply *reply = NULL; |
d3fb4dea | 341 | |
ed85b771 | 342 | debug(11, 3, "httpProcessReplyHeader: key '%s'\n", entry->key); |
f5558c95 | 343 | |
30a4f2a8 | 344 | if (httpState->reply_hdr == NULL) { |
345 | httpState->reply_hdr = get_free_8k_page(); | |
346 | memset(httpState->reply_hdr, '\0', 8192); | |
f5558c95 | 347 | } |
30a4f2a8 | 348 | if (httpState->reply_hdr_state == 0) { |
349 | hdr_len = strlen(httpState->reply_hdr); | |
ed85b771 | 350 | room = 8191 - hdr_len; |
30a4f2a8 | 351 | strncat(httpState->reply_hdr, buf, room < size ? room : size); |
d3fb4dea | 352 | hdr_len += room < size ? room : size; |
30a4f2a8 | 353 | if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { |
60bf30cb | 354 | debug(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", entry->key); |
30a4f2a8 | 355 | httpState->reply_hdr_state += 2; |
b15fe823 | 356 | entry->mem_obj->reply->code = 555; |
ed85b771 | 357 | return; |
d3fb4dea | 358 | } |
d1a43e28 | 359 | t = httpState->reply_hdr + hdr_len; |
360 | /* headers can be incomplete only if object still arriving */ | |
f86a6a46 | 361 | if (!httpState->eof) |
d1a43e28 | 362 | if ((t = mime_headers_end(httpState->reply_hdr)) == NULL) |
363 | return; /* headers not complete */ | |
2285407f | 364 | *t = '\0'; |
365 | reply = entry->mem_obj->reply; | |
30a4f2a8 | 366 | reply->hdr_sz = t - httpState->reply_hdr; |
2285407f | 367 | debug(11, 7, "httpProcessReplyHeader: hdr_sz = %d\n", reply->hdr_sz); |
30a4f2a8 | 368 | httpState->reply_hdr_state++; |
f5558c95 | 369 | } |
30a4f2a8 | 370 | if (httpState->reply_hdr_state == 1) { |
371 | httpState->reply_hdr_state++; | |
019dd986 | 372 | debug(11, 9, "GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", |
30a4f2a8 | 373 | httpState->reply_hdr); |
374 | /* Parse headers into reply structure */ | |
48f44632 | 375 | httpParseReplyHeaders(httpState->reply_hdr, reply); |
38792624 | 376 | timestampsSet(entry); |
30a4f2a8 | 377 | /* Check if object is cacheable or not based on reply code */ |
2285407f | 378 | if (reply->code) |
8e352276 | 379 | debug(11, 3, "httpProcessReplyHeader: HTTP CODE: %d\n", reply->code); |
2285407f | 380 | switch (reply->code) { |
30a4f2a8 | 381 | /* Responses that are cacheable */ |
f5558c95 | 382 | case 200: /* OK */ |
4e38e700 | 383 | case 203: /* Non-Authoritative Information */ |
384 | case 300: /* Multiple Choices */ | |
f5558c95 | 385 | case 301: /* Moved Permanently */ |
4e38e700 | 386 | case 410: /* Gone */ |
30a4f2a8 | 387 | /* don't cache objects from neighbors w/o LMT, Date, or Expires */ |
6fb52f6c | 388 | if (EBIT_TEST(reply->cache_control, SCC_PRIVATE)) |
caebbe00 | 389 | httpMakePrivate(entry); |
6fb52f6c | 390 | else if (EBIT_TEST(reply->cache_control, SCC_NOCACHE)) |
caebbe00 | 391 | httpMakePrivate(entry); |
392 | else if (*reply->date) | |
30a4f2a8 | 393 | httpMakePublic(entry); |
394 | else if (*reply->last_modified) | |
395 | httpMakePublic(entry); | |
396 | else if (!httpState->neighbor) | |
397 | httpMakePublic(entry); | |
398 | else if (*reply->expires) | |
399 | httpMakePublic(entry); | |
af00901c | 400 | else if (entry->mem_obj->request->protocol != PROTO_HTTP) |
401 | /* XXX Remove this check after a while. DW 8/21/96 | |
402 | * We won't keep some FTP objects from neighbors running | |
403 | * 1.0.8 or earlier because their ftpget's don't | |
404 | * add a Date: field */ | |
405 | httpMakePublic(entry); | |
30a4f2a8 | 406 | else |
407 | httpMakePrivate(entry); | |
408 | break; | |
409 | /* Responses that only are cacheable if the server says so */ | |
410 | case 302: /* Moved temporarily */ | |
411 | if (*reply->expires) | |
412 | httpMakePublic(entry); | |
413 | else | |
414 | httpMakePrivate(entry); | |
f5558c95 | 415 | break; |
30a4f2a8 | 416 | /* Errors can be negatively cached */ |
417 | case 204: /* No Content */ | |
418 | case 305: /* Use Proxy (proxy redirect) */ | |
419 | case 400: /* Bad Request */ | |
420 | case 403: /* Forbidden */ | |
421 | case 404: /* Not Found */ | |
422 | case 405: /* Method Now Allowed */ | |
423 | case 414: /* Request-URI Too Long */ | |
424 | case 500: /* Internal Server Error */ | |
425 | case 501: /* Not Implemented */ | |
426 | case 502: /* Bad Gateway */ | |
427 | case 503: /* Service Unavailable */ | |
428 | case 504: /* Gateway Timeout */ | |
851eeef7 | 429 | httpCacheNegatively(entry); |
30a4f2a8 | 430 | break; |
431 | /* Some responses can never be cached */ | |
432 | case 303: /* See Other */ | |
234967c9 | 433 | case 304: /* Not Modified */ |
4e38e700 | 434 | case 401: /* Unauthorized */ |
435 | case 407: /* Proxy Authentication Required */ | |
30a4f2a8 | 436 | default: /* Unknown status code */ |
437 | httpMakePrivate(entry); | |
4e38e700 | 438 | break; |
f5558c95 | 439 | } |
440 | } | |
441 | } | |
442 | ||
090089c4 | 443 | |
444 | /* This will be called when data is ready to be read from fd. Read until | |
445 | * error or connection closed. */ | |
f5558c95 | 446 | /* XXX this function is too long! */ |
b8d8561b | 447 | static void |
b177367b | 448 | httpReadReply(int fd, void *data) |
090089c4 | 449 | { |
b177367b | 450 | HttpStateData *httpState = data; |
95d659f0 | 451 | LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); |
090089c4 | 452 | int len; |
30a4f2a8 | 453 | int bin; |
090089c4 | 454 | int clen; |
455 | int off; | |
456 | StoreEntry *entry = NULL; | |
457 | ||
30a4f2a8 | 458 | entry = httpState->entry; |
234967c9 | 459 | if (entry->flag & DELETE_BEHIND && !storeClientWaiting(entry)) { |
460 | /* we can terminate connection right now */ | |
461 | squid_error_entry(entry, ERR_NO_CLIENTS_BIG_OBJ, NULL); | |
462 | comm_close(fd); | |
463 | return; | |
464 | } | |
465 | /* check if we want to defer reading */ | |
466 | clen = entry->mem_obj->e_current_len; | |
467 | off = storeGetLowestReaderOffset(entry); | |
468 | if ((clen - off) > HTTP_DELETE_GAP) { | |
30a4f2a8 | 469 | if (entry->flag & CLIENT_ABORT_REQUEST) { |
470 | squid_error_entry(entry, ERR_CLIENT_ABORT, NULL); | |
471 | comm_close(fd); | |
472 | return; | |
473 | } | |
474 | IOStats.Http.reads_deferred++; | |
234967c9 | 475 | debug(11, 3, "httpReadReply: Read deferred for Object: %s\n", |
476 | entry->url); | |
477 | debug(11, 3, " Current Gap: %d bytes\n", clen - off); | |
478 | /* reschedule, so it will be automatically reactivated | |
479 | * when Gap is big enough. */ | |
b177367b | 480 | commSetSelect(fd, |
234967c9 | 481 | COMM_SELECT_READ, |
b177367b | 482 | httpReadReply, |
483 | (void *) httpState, 0); | |
30a4f2a8 | 484 | /* disable read timeout until we are below the GAP */ |
b177367b | 485 | commSetSelect(fd, |
234967c9 | 486 | COMM_SELECT_TIMEOUT, |
b177367b | 487 | NULL, |
234967c9 | 488 | (void *) NULL, |
489 | (time_t) 0); | |
56fa4cad | 490 | if (!BIT_TEST(entry->flag, READ_DEFERRED)) { |
491 | comm_set_fd_lifetime(fd, 3600); /* limit during deferring */ | |
492 | BIT_SET(entry->flag, READ_DEFERRED); | |
493 | } | |
234967c9 | 494 | /* dont try reading again for a while */ |
b6f794d6 | 495 | comm_set_stall(fd, Config.stallDelay); |
234967c9 | 496 | return; |
56fa4cad | 497 | } else { |
498 | BIT_RESET(entry->flag, READ_DEFERRED); | |
090089c4 | 499 | } |
1513873c | 500 | errno = 0; |
30a4f2a8 | 501 | len = read(fd, buf, SQUID_TCP_SO_RCVBUF); |
019dd986 | 502 | debug(11, 5, "httpReadReply: FD %d: len %d.\n", fd, len); |
30a4f2a8 | 503 | comm_set_fd_lifetime(fd, 86400); /* extend after good read */ |
504 | if (len > 0) { | |
4a63c85f | 505 | IOStats.Http.reads++; |
30a4f2a8 | 506 | for (clen = len - 1, bin = 0; clen; bin++) |
507 | clen >>= 1; | |
508 | IOStats.Http.read_hist[bin]++; | |
509 | } | |
ba718c8f | 510 | if (len < 0) { |
881f7a6c | 511 | debug(50, 2, "httpReadReply: FD %d: read failure: %s.\n", |
090089c4 | 512 | fd, xstrerror()); |
ba718c8f | 513 | if (errno == EAGAIN || errno == EWOULDBLOCK) { |
1513873c | 514 | /* reinstall handlers */ |
6fe6313d | 515 | /* XXX This may loop forever */ |
b177367b | 516 | commSetSelect(fd, COMM_SELECT_READ, |
517 | httpReadReply, (void *) httpState, 0); | |
518 | commSetSelect(fd, COMM_SELECT_TIMEOUT, | |
519 | httpReadReplyTimeout, (void *) httpState, Config.readTimeout); | |
090089c4 | 520 | } else { |
1c481e00 | 521 | BIT_RESET(entry->flag, ENTRY_CACHABLE); |
2daae136 | 522 | storeReleaseRequest(entry); |
b8de7ebe | 523 | squid_error_entry(entry, ERR_READ_ERROR, xstrerror()); |
0d4d4170 | 524 | comm_close(fd); |
090089c4 | 525 | } |
ba718c8f | 526 | } else if (len == 0 && entry->mem_obj->e_current_len == 0) { |
f86a6a46 | 527 | httpState->eof = 1; |
b8de7ebe | 528 | squid_error_entry(entry, |
ba718c8f | 529 | ERR_ZERO_SIZE_OBJECT, |
530 | errno ? xstrerror() : NULL); | |
0d4d4170 | 531 | comm_close(fd); |
090089c4 | 532 | } else if (len == 0) { |
533 | /* Connection closed; retrieval done. */ | |
f86a6a46 | 534 | httpState->eof = 1; |
d1a43e28 | 535 | if (httpState->reply_hdr_state < 2) |
536 | httpProcessReplyHeader(httpState, buf, len); | |
537 | storeAppend(entry, buf, len); /* invoke handlers! */ | |
538 | storeComplete(entry); /* deallocates mem_obj->request */ | |
0d4d4170 | 539 | comm_close(fd); |
090089c4 | 540 | } else if (entry->flag & CLIENT_ABORT_REQUEST) { |
541 | /* append the last bit of info we get */ | |
542 | storeAppend(entry, buf, len); | |
b8de7ebe | 543 | squid_error_entry(entry, ERR_CLIENT_ABORT, NULL); |
0d4d4170 | 544 | comm_close(fd); |
090089c4 | 545 | } else { |
d1a43e28 | 546 | if (httpState->reply_hdr_state < 2) |
30a4f2a8 | 547 | httpProcessReplyHeader(httpState, buf, len); |
620da955 | 548 | storeAppend(entry, buf, len); |
b177367b | 549 | commSetSelect(fd, |
ba718c8f | 550 | COMM_SELECT_READ, |
b177367b | 551 | httpReadReply, |
552 | (void *) httpState, 0); | |
553 | commSetSelect(fd, | |
ba718c8f | 554 | COMM_SELECT_TIMEOUT, |
b177367b | 555 | httpReadReplyTimeout, |
30a4f2a8 | 556 | (void *) httpState, |
b6f794d6 | 557 | Config.readTimeout); |
090089c4 | 558 | } |
559 | } | |
560 | ||
561 | /* This will be called when request write is complete. Schedule read of | |
562 | * reply. */ | |
b8d8561b | 563 | static void |
564 | httpSendComplete(int fd, char *buf, int size, int errflag, void *data) | |
090089c4 | 565 | { |
30a4f2a8 | 566 | HttpStateData *httpState = data; |
090089c4 | 567 | StoreEntry *entry = NULL; |
568 | ||
30a4f2a8 | 569 | entry = httpState->entry; |
019dd986 | 570 | debug(11, 5, "httpSendComplete: FD %d: size %d: errflag %d.\n", |
090089c4 | 571 | fd, size, errflag); |
572 | ||
090089c4 | 573 | if (errflag) { |
b8de7ebe | 574 | squid_error_entry(entry, ERR_CONNECT_FAIL, xstrerror()); |
0d4d4170 | 575 | comm_close(fd); |
090089c4 | 576 | return; |
577 | } else { | |
578 | /* Schedule read reply. */ | |
b177367b | 579 | commSetSelect(fd, |
019dd986 | 580 | COMM_SELECT_READ, |
b177367b | 581 | httpReadReply, |
582 | (void *) httpState, 0); | |
583 | commSetSelect(fd, | |
019dd986 | 584 | COMM_SELECT_TIMEOUT, |
b177367b | 585 | httpReadReplyTimeout, |
30a4f2a8 | 586 | (void *) httpState, |
b6f794d6 | 587 | Config.readTimeout); |
30a4f2a8 | 588 | comm_set_fd_lifetime(fd, 86400); /* extend lifetime */ |
090089c4 | 589 | } |
590 | } | |
591 | ||
592 | /* This will be called when connect completes. Write request. */ | |
b8d8561b | 593 | static void |
b177367b | 594 | httpSendRequest(int fd, void *data) |
090089c4 | 595 | { |
b177367b | 596 | HttpStateData *httpState = data; |
090089c4 | 597 | char *xbuf = NULL; |
598 | char *ybuf = NULL; | |
599 | char *buf = NULL; | |
600 | char *t = NULL; | |
0ee4272b | 601 | const char *const crlf = "\r\n"; |
090089c4 | 602 | int len = 0; |
603 | int buflen; | |
30a4f2a8 | 604 | request_t *req = httpState->request; |
0ee4272b | 605 | const char *Method = RequestMethodStr[req->method]; |
9864ee44 | 606 | int buftype = 0; |
620da955 | 607 | StoreEntry *entry = httpState->entry; |
f78dec53 | 608 | int saw_host = 0; |
90050993 | 609 | int did_ims = 0; |
48f44632 | 610 | int saw_max_age = 0; |
090089c4 | 611 | |
30a4f2a8 | 612 | debug(11, 5, "httpSendRequest: FD %d: httpState %p.\n", fd, httpState); |
7111c86a | 613 | buflen = strlen(Method) + strlen(req->urlpath); |
30a4f2a8 | 614 | if (httpState->req_hdr) |
4768dbe0 | 615 | buflen += httpState->req_hdr_sz + 1; |
090089c4 | 616 | buflen += 512; /* lots of extra */ |
617 | ||
78657218 | 618 | if ((req->method == METHOD_POST || req->method == METHOD_PUT)) { |
619 | debug_trap("httpSendRequest: should not be handling POST/PUT request"); | |
620 | return; | |
090089c4 | 621 | } |
30a4f2a8 | 622 | if (buflen < DISK_PAGE_SIZE) { |
9864ee44 | 623 | buf = get_free_8k_page(); |
624 | memset(buf, '\0', buflen); | |
625 | buftype = BUF_TYPE_8K; | |
30a4f2a8 | 626 | } else { |
9864ee44 | 627 | buf = xcalloc(buflen, 1); |
628 | buftype = BUF_TYPE_MALLOC; | |
090089c4 | 629 | } |
090089c4 | 630 | |
30a4f2a8 | 631 | sprintf(buf, "%s %s HTTP/1.0\r\n", |
632 | Method, | |
633 | *req->urlpath ? req->urlpath : "/"); | |
090089c4 | 634 | len = strlen(buf); |
90050993 | 635 | /* Add IMS header */ |
636 | if (entry->lastmod && req->method == METHOD_GET) { | |
637 | debug(11, 3, "httpSendRequest: Adding IMS: %s\r\n", | |
638 | mkrfc1123(entry->lastmod)); | |
639 | ybuf = get_free_4k_page(); | |
640 | sprintf(ybuf, "If-Modified-Since: %s\r\n", mkrfc1123(entry->lastmod)); | |
641 | strcat(buf, ybuf); | |
642 | len += strlen(ybuf); | |
643 | put_free_4k_page(ybuf); | |
644 | did_ims = 1; | |
645 | } | |
30a4f2a8 | 646 | if (httpState->req_hdr) { /* we have to parse the request header */ |
647 | xbuf = xstrdup(httpState->req_hdr); | |
090089c4 | 648 | for (t = strtok(xbuf, crlf); t; t = strtok(NULL, crlf)) { |
620da955 | 649 | if (strncasecmp(t, "Connection:", 11) == 0) |
af00901c | 650 | continue; |
f78dec53 | 651 | if (strncasecmp(t, "Host:", 5) == 0) |
652 | saw_host = 1; | |
48f44632 | 653 | if (strncasecmp(t, "Cache-control: Max-age", 5) == 0) |
654 | saw_max_age = 1; | |
90050993 | 655 | if (did_ims && !strncasecmp(t, "If-Modified-Since:", 18)) |
656 | continue; | |
090089c4 | 657 | if (len + (int) strlen(t) > buflen - 10) |
658 | continue; | |
659 | strcat(buf, t); | |
660 | strcat(buf, crlf); | |
661 | len += strlen(t) + 2; | |
662 | } | |
663 | xfree(xbuf); | |
090089c4 | 664 | } |
61f7f59b | 665 | /* Add Via: header */ |
666 | strcat(buf, ViaString); | |
667 | len += strlen(ViaString); | |
daddb143 | 668 | |
f78dec53 | 669 | /* Add Host: header */ |
313c1c28 | 670 | /* Don't add Host: if proxying, mainly because req->host is |
48f44632 | 671 | * the name of the proxy, not the origin :-( */ |
313c1c28 | 672 | if (!saw_host && !BIT_TEST(req->flags, REQ_PROXYING)) { |
f78dec53 | 673 | ybuf = get_free_4k_page(); |
674 | sprintf(ybuf, "Host: %s\r\n", req->host); | |
675 | strcat(buf, ybuf); | |
676 | len += strlen(ybuf); | |
677 | put_free_4k_page(ybuf); | |
678 | } | |
48f44632 | 679 | if (!saw_max_age) { |
680 | ybuf = get_free_4k_page(); | |
681 | sprintf(ybuf, "Cache-control: Max-age=%d\r\n", | |
682 | (int) getMaxAge(entry->url)); | |
683 | strcat(buf, ybuf); | |
684 | len += strlen(ybuf); | |
685 | put_free_4k_page(ybuf); | |
686 | } | |
090089c4 | 687 | strcat(buf, crlf); |
688 | len += 2; | |
67508012 | 689 | debug(11, 6, "httpSendRequest: FD %d:\n%s\n", fd, buf); |
30a4f2a8 | 690 | comm_write(fd, |
14e59844 | 691 | buf, |
692 | len, | |
693 | 30, | |
694 | httpSendComplete, | |
9864ee44 | 695 | httpState, |
696 | buftype == BUF_TYPE_8K ? put_free_8k_page : xfree); | |
090089c4 | 697 | } |
698 | ||
b8d8561b | 699 | int |
fe4e214f | 700 | proxyhttpStart(edge * e, const char *url, StoreEntry * entry) |
090089c4 | 701 | { |
f5558c95 | 702 | int sock; |
30a4f2a8 | 703 | HttpStateData *httpState = NULL; |
7111c86a | 704 | request_t *request = NULL; |
090089c4 | 705 | |
7111c86a | 706 | debug(11, 3, "proxyhttpStart: \"%s %s\"\n", |
efbb0a25 | 707 | RequestMethodStr[entry->method], url); |
019dd986 | 708 | debug(11, 10, "proxyhttpStart: HTTP request header:\n%s\n", |
22e4fa85 | 709 | entry->mem_obj->mime_hdr); |
090089c4 | 710 | |
30a4f2a8 | 711 | if (e->options & NEIGHBOR_PROXY_ONLY) |
090089c4 | 712 | storeStartDeleteBehind(entry); |
713 | ||
714 | /* Create socket. */ | |
16b204c4 | 715 | sock = comm_open(SOCK_STREAM, |
716 | 0, | |
717 | Config.Addrs.tcp_outgoing, | |
718 | 0, | |
719 | COMM_NONBLOCKING, | |
720 | url); | |
090089c4 | 721 | if (sock == COMM_ERROR) { |
019dd986 | 722 | debug(11, 4, "proxyhttpStart: Failed because we're out of sockets.\n"); |
b8de7ebe | 723 | squid_error_entry(entry, ERR_NO_FDS, xstrerror()); |
090089c4 | 724 | return COMM_ERROR; |
725 | } | |
30a4f2a8 | 726 | httpState = xcalloc(1, sizeof(HttpStateData)); |
727 | storeLockObject(httpState->entry = entry, NULL, NULL); | |
728 | httpState->req_hdr = entry->mem_obj->mime_hdr; | |
4768dbe0 | 729 | httpState->req_hdr_sz = entry->mem_obj->mime_hdr_sz; |
30a4f2a8 | 730 | request = get_free_request_t(); |
731 | httpState->request = requestLink(request); | |
732 | httpState->neighbor = e; | |
0d4d4170 | 733 | /* register the handler to free HTTP state data when the FD closes */ |
30a4f2a8 | 734 | comm_add_close_handler(sock, |
b177367b | 735 | httpStateFree, |
30a4f2a8 | 736 | (void *) httpState); |
784213dc | 737 | request->method = entry->method; |
d5aa0e3b | 738 | xstrncpy(request->host, e->host, SQUIDHOSTNAMELEN); |
30a4f2a8 | 739 | request->port = e->http_port; |
d5aa0e3b | 740 | xstrncpy(request->urlpath, url, MAX_URL); |
313c1c28 | 741 | BIT_SET(request->flags, REQ_PROXYING); |
a13d9042 | 742 | ipcache_nbgethostbyname(request->host, |
743 | sock, | |
b15fe823 | 744 | httpConnect, |
a13d9042 | 745 | httpState); |
746 | return COMM_OK; | |
747 | } | |
748 | ||
b15fe823 | 749 | static void |
fe4e214f | 750 | httpConnect(int fd, const ipcache_addrs * ia, void *data) |
a13d9042 | 751 | { |
752 | HttpStateData *httpState = data; | |
753 | request_t *request = httpState->request; | |
754 | StoreEntry *entry = httpState->entry; | |
e5f6c5c2 | 755 | if (ia == NULL) { |
a13d9042 | 756 | debug(11, 4, "httpConnect: Unknown host: %s\n", request->host); |
b8de7ebe | 757 | squid_error_entry(entry, ERR_DNS_FAIL, dns_error_message); |
a13d9042 | 758 | comm_close(fd); |
b15fe823 | 759 | return; |
090089c4 | 760 | } |
761 | /* Open connection. */ | |
e5f6c5c2 | 762 | httpState->connectState.fd = fd; |
763 | httpState->connectState.host = request->host; | |
764 | httpState->connectState.port = request->port; | |
765 | httpState->connectState.handler = httpConnectDone; | |
766 | httpState->connectState.data = httpState; | |
767 | comm_nbconnect(fd, &httpState->connectState); | |
768 | } | |
769 | ||
770 | static void | |
771 | httpConnectDone(int fd, int status, void *data) | |
772 | { | |
773 | HttpStateData *httpState = data; | |
774 | request_t *request = httpState->request; | |
775 | StoreEntry *entry = httpState->entry; | |
776 | edge *e = NULL; | |
777 | if (status != COMM_OK) { | |
778 | if ((e = httpState->neighbor)) { | |
779 | e->last_fail_time = squid_curtime; | |
780 | e->neighbor_up = 0; | |
090089c4 | 781 | } |
e5f6c5c2 | 782 | squid_error_entry(entry, ERR_CONNECT_FAIL, xstrerror()); |
783 | comm_close(fd); | |
784 | } else { | |
785 | /* Install connection complete handler. */ | |
786 | if (opt_no_ipcache) | |
787 | ipcacheInvalidate(request->host); | |
788 | fd_note(fd, entry->url); | |
b177367b | 789 | commSetSelect(fd, COMM_SELECT_LIFETIME, |
790 | httpLifetimeExpire, (void *) httpState, 0); | |
791 | commSetSelect(fd, COMM_SELECT_WRITE, | |
792 | httpSendRequest, (void *) httpState, 0); | |
ef2d27ff | 793 | if (Config.vizHackAddr.sin_port) |
794 | vizHackSendPkt(&httpState->connectState.S, 2); | |
090089c4 | 795 | } |
090089c4 | 796 | } |
797 | ||
b8d8561b | 798 | int |
4768dbe0 | 799 | httpStart(int unusedfd, |
800 | char *url, | |
801 | request_t * request, | |
802 | char *req_hdr, | |
803 | int req_hdr_sz, | |
804 | StoreEntry * entry) | |
090089c4 | 805 | { |
806 | /* Create state structure. */ | |
a13d9042 | 807 | int sock; |
30a4f2a8 | 808 | HttpStateData *httpState = NULL; |
090089c4 | 809 | |
7111c86a | 810 | debug(11, 3, "httpStart: \"%s %s\"\n", |
811 | RequestMethodStr[request->method], url); | |
019dd986 | 812 | debug(11, 10, "httpStart: req_hdr '%s'\n", req_hdr); |
090089c4 | 813 | |
090089c4 | 814 | /* Create socket. */ |
16b204c4 | 815 | sock = comm_open(SOCK_STREAM, |
816 | 0, | |
817 | Config.Addrs.tcp_outgoing, | |
818 | 0, | |
819 | COMM_NONBLOCKING, | |
820 | url); | |
090089c4 | 821 | if (sock == COMM_ERROR) { |
019dd986 | 822 | debug(11, 4, "httpStart: Failed because we're out of sockets.\n"); |
b8de7ebe | 823 | squid_error_entry(entry, ERR_NO_FDS, xstrerror()); |
090089c4 | 824 | return COMM_ERROR; |
825 | } | |
30a4f2a8 | 826 | httpState = xcalloc(1, sizeof(HttpStateData)); |
827 | storeLockObject(httpState->entry = entry, NULL, NULL); | |
828 | httpState->req_hdr = req_hdr; | |
4768dbe0 | 829 | httpState->req_hdr_sz = req_hdr_sz; |
30a4f2a8 | 830 | httpState->request = requestLink(request); |
831 | comm_add_close_handler(sock, | |
b177367b | 832 | httpStateFree, |
30a4f2a8 | 833 | (void *) httpState); |
a13d9042 | 834 | ipcache_nbgethostbyname(request->host, |
835 | sock, | |
836 | httpConnect, | |
837 | httpState); | |
090089c4 | 838 | return COMM_OK; |
839 | } | |
30a4f2a8 | 840 | |
b8d8561b | 841 | void |
842 | httpReplyHeaderStats(StoreEntry * entry) | |
30a4f2a8 | 843 | { |
6fb52f6c | 844 | http_server_cc_t i; |
30a4f2a8 | 845 | storeAppendPrintf(entry, open_bracket); |
6fb52f6c | 846 | storeAppendPrintf(entry, "{HTTP Reply Headers:}\n"); |
847 | storeAppendPrintf(entry, "{ Headers parsed: %d}\n", | |
30a4f2a8 | 848 | ReplyHeaderStats.parsed); |
6fb52f6c | 849 | storeAppendPrintf(entry, "{ Date: %d}\n", |
30a4f2a8 | 850 | ReplyHeaderStats.date); |
6fb52f6c | 851 | storeAppendPrintf(entry, "{ Last-Modified: %d}\n", |
30a4f2a8 | 852 | ReplyHeaderStats.lm); |
6fb52f6c | 853 | storeAppendPrintf(entry, "{ Expires: %d}\n", |
30a4f2a8 | 854 | ReplyHeaderStats.exp); |
6fb52f6c | 855 | storeAppendPrintf(entry, "{ Content-Type: %d}\n", |
30a4f2a8 | 856 | ReplyHeaderStats.ctype); |
6fb52f6c | 857 | storeAppendPrintf(entry, "{ Content-Length: %d}\n", |
30a4f2a8 | 858 | ReplyHeaderStats.clen); |
4db43fab | 859 | for (i = SCC_PUBLIC; i < SCC_ENUM_END; i++) |
6fb52f6c | 860 | storeAppendPrintf(entry, "{Cache-Control %7.7s: %d}\n", |
861 | HttpServerCCStr[i], | |
862 | ReplyHeaderStats.cc[i]); | |
30a4f2a8 | 863 | storeAppendPrintf(entry, close_bracket); |
864 | } |