]>
Commit | Line | Data |
---|---|---|
30a4f2a8 | 1 | /* |
38792624 | 2 | * $Id: http.cc,v 1.89 1996/10/28 07:44:22 wessels Exp $ |
30a4f2a8 | 3 | * |
4 | * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) | |
5 | * AUTHOR: Harvest Derived | |
6 | * | |
7 | * SQUID Internet Object Cache http://www.nlanr.net/Squid/ | |
8 | * -------------------------------------------------------- | |
9 | * | |
10 | * Squid is the result of efforts by numerous individuals from the | |
11 | * Internet community. Development is led by Duane Wessels of the | |
12 | * National Laboratory for Applied Network Research and funded by | |
13 | * the National Science Foundation. | |
14 | * | |
15 | * This program is free software; you can redistribute it and/or modify | |
16 | * it under the terms of the GNU General Public License as published by | |
17 | * the Free Software Foundation; either version 2 of the License, or | |
18 | * (at your option) any later version. | |
19 | * | |
20 | * This program is distributed in the hope that it will be useful, | |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
24 | * | |
25 | * You should have received a copy of the GNU General Public License | |
26 | * along with this program; if not, write to the Free Software | |
27 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
28 | * | |
29 | */ | |
019dd986 | 30 | |
31 | /* | |
30a4f2a8 | 32 | * Copyright (c) 1994, 1995. All rights reserved. |
33 | * | |
34 | * The Harvest software was developed by the Internet Research Task | |
35 | * Force Research Group on Resource Discovery (IRTF-RD): | |
36 | * | |
37 | * Mic Bowman of Transarc Corporation. | |
38 | * Peter Danzig of the University of Southern California. | |
39 | * Darren R. Hardy of the University of Colorado at Boulder. | |
40 | * Udi Manber of the University of Arizona. | |
41 | * Michael F. Schwartz of the University of Colorado at Boulder. | |
42 | * Duane Wessels of the University of Colorado at Boulder. | |
43 | * | |
44 | * This copyright notice applies to software in the Harvest | |
45 | * ``src/'' directory only. Users should consult the individual | |
46 | * copyright notices in the ``components/'' subdirectories for | |
47 | * copyright information about other software bundled with the | |
48 | * Harvest source code distribution. | |
49 | * | |
50 | * TERMS OF USE | |
51 | * | |
52 | * The Harvest software may be used and re-distributed without | |
53 | * charge, provided that the software origin and research team are | |
54 | * cited in any use of the system. Most commonly this is | |
55 | * accomplished by including a link to the Harvest Home Page | |
56 | * (http://harvest.cs.colorado.edu/) from the query page of any | |
57 | * Broker you deploy, as well as in the query result pages. These | |
58 | * links are generated automatically by the standard Broker | |
59 | * software distribution. | |
60 | * | |
61 | * The Harvest software is provided ``as is'', without express or | |
62 | * implied warranty, and with no support nor obligation to assist | |
63 | * in its use, correction, modification or enhancement. We assume | |
64 | * no liability with respect to the infringement of copyrights, | |
65 | * trade secrets, or any patents, and are not responsible for | |
66 | * consequential damages. Proper use of the Harvest software is | |
67 | * entirely the responsibility of the user. | |
68 | * | |
69 | * DERIVATIVE WORKS | |
70 | * | |
71 | * Users may make derivative works from the Harvest software, subject | |
72 | * to the following constraints: | |
73 | * | |
74 | * - You must include the above copyright notice and these | |
75 | * accompanying paragraphs in all forms of derivative works, | |
76 | * and any documentation and other materials related to such | |
77 | * distribution and use acknowledge that the software was | |
78 | * developed at the above institutions. | |
79 | * | |
80 | * - You must notify IRTF-RD regarding your distribution of | |
81 | * the derivative work. | |
82 | * | |
83 | * - You must clearly notify users that your are distributing | |
84 | * a modified version and not the original Harvest software. | |
85 | * | |
86 | * - Any derivative product is also subject to these copyright | |
87 | * and use restrictions. | |
88 | * | |
89 | * Note that the Harvest software is NOT in the public domain. We | |
90 | * retain copyright, as specified above. | |
91 | * | |
92 | * HISTORY OF FREE SOFTWARE STATUS | |
93 | * | |
94 | * Originally we required sites to license the software in cases | |
95 | * where they were going to build commercial products/services | |
96 | * around Harvest. In June 1995 we changed this policy. We now | |
97 | * allow people to use the core Harvest software (the code found in | |
98 | * the Harvest ``src/'' directory) for free. We made this change | |
99 | * in the interest of encouraging the widest possible deployment of | |
100 | * the technology. The Harvest software is really a reference | |
101 | * implementation of a set of protocols and formats, some of which | |
102 | * we intend to standardize. We encourage commercial | |
103 | * re-implementations of code complying to this set of standards. | |
019dd986 | 104 | */ |
44a47c6e | 105 | |
106 | #include "squid.h" | |
090089c4 | 107 | |
234967c9 | 108 | #define HTTP_DELETE_GAP (1<<18) |
090089c4 | 109 | |
24382924 | 110 | static struct { |
30a4f2a8 | 111 | int parsed; |
112 | int date; | |
113 | int lm; | |
114 | int exp; | |
115 | int clen; | |
116 | int ctype; | |
117 | } ReplyHeaderStats; | |
090089c4 | 118 | |
b177367b | 119 | static void httpStateFree _PARAMS((int fd, void *)); |
120 | static void httpReadReplyTimeout _PARAMS((int fd, void *)); | |
121 | static void httpLifetimeExpire _PARAMS((int fd, void *)); | |
67508012 | 122 | static void httpMakePublic _PARAMS((StoreEntry *)); |
123 | static void httpMakePrivate _PARAMS((StoreEntry *)); | |
124 | static void httpCacheNegatively _PARAMS((StoreEntry *)); | |
b177367b | 125 | static void httpReadReply _PARAMS((int fd, void *)); |
67508012 | 126 | static void httpSendComplete _PARAMS((int fd, char *, int, int, void *)); |
b177367b | 127 | static void httpSendRequest _PARAMS((int fd, void *)); |
e5f6c5c2 | 128 | static void httpConnect _PARAMS((int fd, ipcache_addrs *, void *)); |
129 | static void httpConnectDone _PARAMS((int fd, int status, void *data)); | |
b8d8561b | 130 | |
b177367b | 131 | static void |
132 | httpStateFree(int fd, void *data) | |
f5558c95 | 133 | { |
b177367b | 134 | HttpStateData *httpState = data; |
0d4d4170 | 135 | if (httpState == NULL) |
b177367b | 136 | return; |
30a4f2a8 | 137 | storeUnlockObject(httpState->entry); |
0d4d4170 | 138 | if (httpState->reply_hdr) { |
139 | put_free_8k_page(httpState->reply_hdr); | |
140 | httpState->reply_hdr = NULL; | |
141 | } | |
30a4f2a8 | 142 | requestUnlink(httpState->request); |
0d4d4170 | 143 | xfree(httpState); |
f5558c95 | 144 | } |
145 | ||
b8d8561b | 146 | int |
147 | httpCachable(char *url, int method) | |
090089c4 | 148 | { |
090089c4 | 149 | /* GET and HEAD are cachable. Others are not. */ |
6eb42cae | 150 | if (method != METHOD_GET && method != METHOD_HEAD) |
090089c4 | 151 | return 0; |
090089c4 | 152 | /* else cachable */ |
153 | return 1; | |
154 | } | |
155 | ||
156 | /* This will be called when timeout on read. */ | |
b8d8561b | 157 | static void |
b177367b | 158 | httpReadReplyTimeout(int fd, void *data) |
090089c4 | 159 | { |
b177367b | 160 | HttpStateData *httpState = data; |
090089c4 | 161 | StoreEntry *entry = NULL; |
162 | ||
30a4f2a8 | 163 | entry = httpState->entry; |
019dd986 | 164 | debug(11, 4, "httpReadReplyTimeout: FD %d: <URL:%s>\n", fd, entry->url); |
b8de7ebe | 165 | squid_error_entry(entry, ERR_READ_TIMEOUT, NULL); |
b177367b | 166 | commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); |
0d4d4170 | 167 | comm_close(fd); |
090089c4 | 168 | } |
169 | ||
170 | /* This will be called when socket lifetime is expired. */ | |
b8d8561b | 171 | static void |
b177367b | 172 | httpLifetimeExpire(int fd, void *data) |
090089c4 | 173 | { |
b177367b | 174 | HttpStateData *httpState = data; |
090089c4 | 175 | StoreEntry *entry = NULL; |
176 | ||
30a4f2a8 | 177 | entry = httpState->entry; |
019dd986 | 178 | debug(11, 4, "httpLifeTimeExpire: FD %d: <URL:%s>\n", fd, entry->url); |
090089c4 | 179 | |
b8de7ebe | 180 | squid_error_entry(entry, ERR_LIFETIME_EXP, NULL); |
b177367b | 181 | commSetSelect(fd, COMM_SELECT_READ | COMM_SELECT_WRITE, NULL, NULL, 0); |
0d4d4170 | 182 | comm_close(fd); |
090089c4 | 183 | } |
184 | ||
30a4f2a8 | 185 | /* This object can be cached for a long time */ |
b8d8561b | 186 | static void |
187 | httpMakePublic(StoreEntry * entry) | |
30a4f2a8 | 188 | { |
1c481e00 | 189 | if (BIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 190 | storeSetPublicKey(entry); |
191 | } | |
192 | ||
193 | /* This object should never be cached at all */ | |
b8d8561b | 194 | static void |
195 | httpMakePrivate(StoreEntry * entry) | |
30a4f2a8 | 196 | { |
197 | storeSetPrivateKey(entry); | |
198 | storeExpireNow(entry); | |
1c481e00 | 199 | BIT_RESET(entry->flag, ENTRY_CACHABLE); |
30a4f2a8 | 200 | storeReleaseRequest(entry); /* delete object when not used */ |
201 | } | |
202 | ||
203 | /* This object may be negatively cached */ | |
b8d8561b | 204 | static void |
205 | httpCacheNegatively(StoreEntry * entry) | |
30a4f2a8 | 206 | { |
79b5cc5f | 207 | storeNegativeCache(entry); |
1c481e00 | 208 | if (BIT_TEST(entry->flag, ENTRY_CACHABLE)) |
30a4f2a8 | 209 | storeSetPublicKey(entry); |
30a4f2a8 | 210 | } |
211 | ||
212 | ||
213 | /* Build a reply structure from HTTP reply headers */ | |
b8d8561b | 214 | void |
215 | httpParseHeaders(char *buf, struct _http_reply *reply) | |
30a4f2a8 | 216 | { |
217 | char *headers = NULL; | |
218 | char *t = NULL; | |
219 | char *s = NULL; | |
220 | ||
221 | ReplyHeaderStats.parsed++; | |
222 | headers = xstrdup(buf); | |
223 | t = strtok(headers, "\n"); | |
224 | while (t) { | |
225 | s = t + strlen(t); | |
226 | while (*s == '\r') | |
227 | *s-- = '\0'; | |
228 | if (!strncasecmp(t, "HTTP", 4)) { | |
229 | sscanf(t + 1, "%lf", &reply->version); | |
230 | if ((t = strchr(t, ' '))) { | |
231 | t++; | |
232 | reply->code = atoi(t); | |
233 | } | |
234 | } else if (!strncasecmp(t, "Content-type:", 13)) { | |
235 | if ((t = strchr(t, ' '))) { | |
236 | t++; | |
237 | strncpy(reply->content_type, t, HTTP_REPLY_FIELD_SZ - 1); | |
238 | ReplyHeaderStats.ctype++; | |
239 | } | |
240 | } else if (!strncasecmp(t, "Content-length:", 15)) { | |
241 | if ((t = strchr(t, ' '))) { | |
242 | t++; | |
243 | reply->content_length = atoi(t); | |
244 | ReplyHeaderStats.clen++; | |
245 | } | |
246 | } else if (!strncasecmp(t, "Date:", 5)) { | |
247 | if ((t = strchr(t, ' '))) { | |
248 | t++; | |
249 | strncpy(reply->date, t, HTTP_REPLY_FIELD_SZ - 1); | |
250 | ReplyHeaderStats.date++; | |
251 | } | |
252 | } else if (!strncasecmp(t, "Expires:", 8)) { | |
253 | if ((t = strchr(t, ' '))) { | |
254 | t++; | |
255 | strncpy(reply->expires, t, HTTP_REPLY_FIELD_SZ - 1); | |
256 | ReplyHeaderStats.exp++; | |
257 | } | |
258 | } else if (!strncasecmp(t, "Last-Modified:", 14)) { | |
259 | if ((t = strchr(t, ' '))) { | |
260 | t++; | |
261 | strncpy(reply->last_modified, t, HTTP_REPLY_FIELD_SZ - 1); | |
262 | ReplyHeaderStats.lm++; | |
263 | } | |
caebbe00 | 264 | } else if (!strncasecmp(t, "Cache-Control:", 14)) { |
265 | if ((t = strtok(NULL, w_space))) { | |
266 | if (!strncasecmp(t, "private", 7)) | |
267 | reply->cache_control |= HTTP_CC_PRIVATE; | |
268 | else if (!strncasecmp(t, "cachable", 8)) | |
269 | reply->cache_control |= HTTP_CC_CACHABLE; | |
270 | else if (!strncasecmp(t, "no-cache", 8)) | |
271 | reply->cache_control |= HTTP_CC_NOCACHE; | |
272 | } | |
30a4f2a8 | 273 | } |
274 | t = strtok(NULL, "\n"); | |
275 | } | |
1c481e00 | 276 | #if LOG_TIMESTAMPS |
277 | fprintf(timestamp_log, "T %9d D %9d L %9d E %9d\n", | |
278 | squid_curtime, | |
cf49df4b | 279 | parse_rfc1123(reply->date), |
280 | parse_rfc1123(reply->last_modified), | |
281 | parse_rfc1123(reply->expires)); | |
1c481e00 | 282 | #endif /* LOG_TIMESTAMPS */ |
30a4f2a8 | 283 | safe_free(headers); |
284 | } | |
285 | ||
090089c4 | 286 | |
b8d8561b | 287 | void |
288 | httpProcessReplyHeader(HttpStateData * httpState, char *buf, int size) | |
f5558c95 | 289 | { |
290 | char *t = NULL; | |
30a4f2a8 | 291 | StoreEntry *entry = httpState->entry; |
d3fb4dea | 292 | int room; |
293 | int hdr_len; | |
2285407f | 294 | struct _http_reply *reply = NULL; |
d3fb4dea | 295 | |
ed85b771 | 296 | debug(11, 3, "httpProcessReplyHeader: key '%s'\n", entry->key); |
f5558c95 | 297 | |
30a4f2a8 | 298 | if (httpState->reply_hdr == NULL) { |
299 | httpState->reply_hdr = get_free_8k_page(); | |
300 | memset(httpState->reply_hdr, '\0', 8192); | |
f5558c95 | 301 | } |
30a4f2a8 | 302 | if (httpState->reply_hdr_state == 0) { |
303 | hdr_len = strlen(httpState->reply_hdr); | |
ed85b771 | 304 | room = 8191 - hdr_len; |
30a4f2a8 | 305 | strncat(httpState->reply_hdr, buf, room < size ? room : size); |
d3fb4dea | 306 | hdr_len += room < size ? room : size; |
30a4f2a8 | 307 | if (hdr_len > 4 && strncmp(httpState->reply_hdr, "HTTP/", 5)) { |
60bf30cb | 308 | debug(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", entry->key); |
30a4f2a8 | 309 | httpState->reply_hdr_state += 2; |
b15fe823 | 310 | entry->mem_obj->reply->code = 555; |
ed85b771 | 311 | return; |
d3fb4dea | 312 | } |
d1a43e28 | 313 | t = httpState->reply_hdr + hdr_len; |
314 | /* headers can be incomplete only if object still arriving */ | |
f86a6a46 | 315 | if (!httpState->eof) |
d1a43e28 | 316 | if ((t = mime_headers_end(httpState->reply_hdr)) == NULL) |
317 | return; /* headers not complete */ | |
2285407f | 318 | *t = '\0'; |
319 | reply = entry->mem_obj->reply; | |
30a4f2a8 | 320 | reply->hdr_sz = t - httpState->reply_hdr; |
2285407f | 321 | debug(11, 7, "httpProcessReplyHeader: hdr_sz = %d\n", reply->hdr_sz); |
30a4f2a8 | 322 | httpState->reply_hdr_state++; |
f5558c95 | 323 | } |
30a4f2a8 | 324 | if (httpState->reply_hdr_state == 1) { |
325 | httpState->reply_hdr_state++; | |
019dd986 | 326 | debug(11, 9, "GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", |
30a4f2a8 | 327 | httpState->reply_hdr); |
328 | /* Parse headers into reply structure */ | |
329 | httpParseHeaders(httpState->reply_hdr, reply); | |
38792624 | 330 | timestampsSet(entry); |
30a4f2a8 | 331 | /* Check if object is cacheable or not based on reply code */ |
2285407f | 332 | if (reply->code) |
8e352276 | 333 | debug(11, 3, "httpProcessReplyHeader: HTTP CODE: %d\n", reply->code); |
2285407f | 334 | switch (reply->code) { |
30a4f2a8 | 335 | /* Responses that are cacheable */ |
f5558c95 | 336 | case 200: /* OK */ |
4e38e700 | 337 | case 203: /* Non-Authoritative Information */ |
338 | case 300: /* Multiple Choices */ | |
f5558c95 | 339 | case 301: /* Moved Permanently */ |
4e38e700 | 340 | case 410: /* Gone */ |
30a4f2a8 | 341 | /* don't cache objects from neighbors w/o LMT, Date, or Expires */ |
caebbe00 | 342 | if (BIT_TEST(reply->cache_control, HTTP_CC_PRIVATE)) |
343 | httpMakePrivate(entry); | |
344 | else if (BIT_TEST(reply->cache_control, HTTP_CC_NOCACHE)) | |
345 | httpMakePrivate(entry); | |
346 | else if (*reply->date) | |
30a4f2a8 | 347 | httpMakePublic(entry); |
348 | else if (*reply->last_modified) | |
349 | httpMakePublic(entry); | |
350 | else if (!httpState->neighbor) | |
351 | httpMakePublic(entry); | |
352 | else if (*reply->expires) | |
353 | httpMakePublic(entry); | |
af00901c | 354 | else if (entry->mem_obj->request->protocol != PROTO_HTTP) |
355 | /* XXX Remove this check after a while. DW 8/21/96 | |
356 | * We won't keep some FTP objects from neighbors running | |
357 | * 1.0.8 or earlier because their ftpget's don't | |
358 | * add a Date: field */ | |
359 | httpMakePublic(entry); | |
30a4f2a8 | 360 | else |
361 | httpMakePrivate(entry); | |
362 | break; | |
363 | /* Responses that only are cacheable if the server says so */ | |
364 | case 302: /* Moved temporarily */ | |
365 | if (*reply->expires) | |
366 | httpMakePublic(entry); | |
367 | else | |
368 | httpMakePrivate(entry); | |
f5558c95 | 369 | break; |
30a4f2a8 | 370 | /* Errors can be negatively cached */ |
371 | case 204: /* No Content */ | |
372 | case 305: /* Use Proxy (proxy redirect) */ | |
373 | case 400: /* Bad Request */ | |
374 | case 403: /* Forbidden */ | |
375 | case 404: /* Not Found */ | |
376 | case 405: /* Method Now Allowed */ | |
377 | case 414: /* Request-URI Too Long */ | |
378 | case 500: /* Internal Server Error */ | |
379 | case 501: /* Not Implemented */ | |
380 | case 502: /* Bad Gateway */ | |
381 | case 503: /* Service Unavailable */ | |
382 | case 504: /* Gateway Timeout */ | |
383 | if (*reply->expires) | |
384 | httpMakePublic(entry); | |
385 | else | |
386 | httpCacheNegatively(entry); | |
387 | break; | |
388 | /* Some responses can never be cached */ | |
389 | case 303: /* See Other */ | |
234967c9 | 390 | case 304: /* Not Modified */ |
4e38e700 | 391 | case 401: /* Unauthorized */ |
392 | case 407: /* Proxy Authentication Required */ | |
30a4f2a8 | 393 | default: /* Unknown status code */ |
394 | httpMakePrivate(entry); | |
4e38e700 | 395 | break; |
f5558c95 | 396 | } |
397 | } | |
398 | } | |
399 | ||
090089c4 | 400 | |
401 | /* This will be called when data is ready to be read from fd. Read until | |
402 | * error or connection closed. */ | |
f5558c95 | 403 | /* XXX this function is too long! */ |
b8d8561b | 404 | static void |
b177367b | 405 | httpReadReply(int fd, void *data) |
090089c4 | 406 | { |
b177367b | 407 | HttpStateData *httpState = data; |
95d659f0 | 408 | LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); |
090089c4 | 409 | int len; |
30a4f2a8 | 410 | int bin; |
090089c4 | 411 | int clen; |
412 | int off; | |
413 | StoreEntry *entry = NULL; | |
414 | ||
30a4f2a8 | 415 | entry = httpState->entry; |
234967c9 | 416 | if (entry->flag & DELETE_BEHIND && !storeClientWaiting(entry)) { |
417 | /* we can terminate connection right now */ | |
418 | squid_error_entry(entry, ERR_NO_CLIENTS_BIG_OBJ, NULL); | |
419 | comm_close(fd); | |
420 | return; | |
421 | } | |
422 | /* check if we want to defer reading */ | |
423 | clen = entry->mem_obj->e_current_len; | |
424 | off = storeGetLowestReaderOffset(entry); | |
425 | if ((clen - off) > HTTP_DELETE_GAP) { | |
30a4f2a8 | 426 | if (entry->flag & CLIENT_ABORT_REQUEST) { |
427 | squid_error_entry(entry, ERR_CLIENT_ABORT, NULL); | |
428 | comm_close(fd); | |
429 | return; | |
430 | } | |
431 | IOStats.Http.reads_deferred++; | |
234967c9 | 432 | debug(11, 3, "httpReadReply: Read deferred for Object: %s\n", |
433 | entry->url); | |
434 | debug(11, 3, " Current Gap: %d bytes\n", clen - off); | |
435 | /* reschedule, so it will be automatically reactivated | |
436 | * when Gap is big enough. */ | |
b177367b | 437 | commSetSelect(fd, |
234967c9 | 438 | COMM_SELECT_READ, |
b177367b | 439 | httpReadReply, |
440 | (void *) httpState, 0); | |
30a4f2a8 | 441 | /* disable read timeout until we are below the GAP */ |
b177367b | 442 | commSetSelect(fd, |
234967c9 | 443 | COMM_SELECT_TIMEOUT, |
b177367b | 444 | NULL, |
234967c9 | 445 | (void *) NULL, |
446 | (time_t) 0); | |
56fa4cad | 447 | if (!BIT_TEST(entry->flag, READ_DEFERRED)) { |
448 | comm_set_fd_lifetime(fd, 3600); /* limit during deferring */ | |
449 | BIT_SET(entry->flag, READ_DEFERRED); | |
450 | } | |
234967c9 | 451 | /* dont try reading again for a while */ |
b6f794d6 | 452 | comm_set_stall(fd, Config.stallDelay); |
234967c9 | 453 | return; |
56fa4cad | 454 | } else { |
455 | BIT_RESET(entry->flag, READ_DEFERRED); | |
090089c4 | 456 | } |
1513873c | 457 | errno = 0; |
30a4f2a8 | 458 | len = read(fd, buf, SQUID_TCP_SO_RCVBUF); |
019dd986 | 459 | debug(11, 5, "httpReadReply: FD %d: len %d.\n", fd, len); |
30a4f2a8 | 460 | comm_set_fd_lifetime(fd, 86400); /* extend after good read */ |
461 | if (len > 0) { | |
4a63c85f | 462 | IOStats.Http.reads++; |
30a4f2a8 | 463 | for (clen = len - 1, bin = 0; clen; bin++) |
464 | clen >>= 1; | |
465 | IOStats.Http.read_hist[bin]++; | |
466 | } | |
ba718c8f | 467 | if (len < 0) { |
019dd986 | 468 | debug(11, 2, "httpReadReply: FD %d: read failure: %s.\n", |
090089c4 | 469 | fd, xstrerror()); |
ba718c8f | 470 | if (errno == EAGAIN || errno == EWOULDBLOCK) { |
1513873c | 471 | /* reinstall handlers */ |
6fe6313d | 472 | /* XXX This may loop forever */ |
b177367b | 473 | commSetSelect(fd, COMM_SELECT_READ, |
474 | httpReadReply, (void *) httpState, 0); | |
475 | commSetSelect(fd, COMM_SELECT_TIMEOUT, | |
476 | httpReadReplyTimeout, (void *) httpState, Config.readTimeout); | |
090089c4 | 477 | } else { |
1c481e00 | 478 | BIT_RESET(entry->flag, ENTRY_CACHABLE); |
2daae136 | 479 | storeReleaseRequest(entry); |
b8de7ebe | 480 | squid_error_entry(entry, ERR_READ_ERROR, xstrerror()); |
0d4d4170 | 481 | comm_close(fd); |
090089c4 | 482 | } |
ba718c8f | 483 | } else if (len == 0 && entry->mem_obj->e_current_len == 0) { |
f86a6a46 | 484 | httpState->eof = 1; |
b8de7ebe | 485 | squid_error_entry(entry, |
ba718c8f | 486 | ERR_ZERO_SIZE_OBJECT, |
487 | errno ? xstrerror() : NULL); | |
0d4d4170 | 488 | comm_close(fd); |
090089c4 | 489 | } else if (len == 0) { |
490 | /* Connection closed; retrieval done. */ | |
f86a6a46 | 491 | httpState->eof = 1; |
d1a43e28 | 492 | if (httpState->reply_hdr_state < 2) |
493 | httpProcessReplyHeader(httpState, buf, len); | |
494 | storeAppend(entry, buf, len); /* invoke handlers! */ | |
495 | storeComplete(entry); /* deallocates mem_obj->request */ | |
0d4d4170 | 496 | comm_close(fd); |
090089c4 | 497 | } else if (entry->flag & CLIENT_ABORT_REQUEST) { |
498 | /* append the last bit of info we get */ | |
499 | storeAppend(entry, buf, len); | |
b8de7ebe | 500 | squid_error_entry(entry, ERR_CLIENT_ABORT, NULL); |
0d4d4170 | 501 | comm_close(fd); |
090089c4 | 502 | } else { |
d1a43e28 | 503 | if (httpState->reply_hdr_state < 2) |
30a4f2a8 | 504 | httpProcessReplyHeader(httpState, buf, len); |
620da955 | 505 | storeAppend(entry, buf, len); |
b177367b | 506 | commSetSelect(fd, |
ba718c8f | 507 | COMM_SELECT_READ, |
b177367b | 508 | httpReadReply, |
509 | (void *) httpState, 0); | |
510 | commSetSelect(fd, | |
ba718c8f | 511 | COMM_SELECT_TIMEOUT, |
b177367b | 512 | httpReadReplyTimeout, |
30a4f2a8 | 513 | (void *) httpState, |
b6f794d6 | 514 | Config.readTimeout); |
090089c4 | 515 | } |
516 | } | |
517 | ||
518 | /* This will be called when request write is complete. Schedule read of | |
519 | * reply. */ | |
b8d8561b | 520 | static void |
521 | httpSendComplete(int fd, char *buf, int size, int errflag, void *data) | |
090089c4 | 522 | { |
30a4f2a8 | 523 | HttpStateData *httpState = data; |
090089c4 | 524 | StoreEntry *entry = NULL; |
525 | ||
30a4f2a8 | 526 | entry = httpState->entry; |
019dd986 | 527 | debug(11, 5, "httpSendComplete: FD %d: size %d: errflag %d.\n", |
090089c4 | 528 | fd, size, errflag); |
529 | ||
090089c4 | 530 | if (errflag) { |
b8de7ebe | 531 | squid_error_entry(entry, ERR_CONNECT_FAIL, xstrerror()); |
0d4d4170 | 532 | comm_close(fd); |
090089c4 | 533 | return; |
534 | } else { | |
535 | /* Schedule read reply. */ | |
b177367b | 536 | commSetSelect(fd, |
019dd986 | 537 | COMM_SELECT_READ, |
b177367b | 538 | httpReadReply, |
539 | (void *) httpState, 0); | |
540 | commSetSelect(fd, | |
019dd986 | 541 | COMM_SELECT_TIMEOUT, |
b177367b | 542 | httpReadReplyTimeout, |
30a4f2a8 | 543 | (void *) httpState, |
b6f794d6 | 544 | Config.readTimeout); |
30a4f2a8 | 545 | comm_set_fd_lifetime(fd, 86400); /* extend lifetime */ |
090089c4 | 546 | } |
547 | } | |
548 | ||
549 | /* This will be called when connect completes. Write request. */ | |
b8d8561b | 550 | static void |
b177367b | 551 | httpSendRequest(int fd, void *data) |
090089c4 | 552 | { |
b177367b | 553 | HttpStateData *httpState = data; |
090089c4 | 554 | char *xbuf = NULL; |
555 | char *ybuf = NULL; | |
556 | char *buf = NULL; | |
557 | char *t = NULL; | |
558 | char *post_buf = NULL; | |
4768dbe0 | 559 | int post_buf_sz = 0; |
090089c4 | 560 | static char *crlf = "\r\n"; |
090089c4 | 561 | int len = 0; |
562 | int buflen; | |
d9766e35 | 563 | int cfd = -1; |
30a4f2a8 | 564 | request_t *req = httpState->request; |
7111c86a | 565 | char *Method = RequestMethodStr[req->method]; |
9864ee44 | 566 | int buftype = 0; |
620da955 | 567 | StoreEntry *entry = httpState->entry; |
f78dec53 | 568 | int saw_host = 0; |
090089c4 | 569 | |
30a4f2a8 | 570 | debug(11, 5, "httpSendRequest: FD %d: httpState %p.\n", fd, httpState); |
7111c86a | 571 | buflen = strlen(Method) + strlen(req->urlpath); |
30a4f2a8 | 572 | if (httpState->req_hdr) |
4768dbe0 | 573 | buflen += httpState->req_hdr_sz + 1; |
090089c4 | 574 | buflen += 512; /* lots of extra */ |
575 | ||
30a4f2a8 | 576 | if ((req->method == METHOD_POST || req->method == METHOD_PUT) && httpState->req_hdr) { |
577 | if ((t = mime_headers_end(httpState->req_hdr))) { | |
4768dbe0 | 578 | post_buf_sz = httpState->req_hdr_sz - (t - httpState->req_hdr); |
579 | post_buf = xmalloc(post_buf_sz + 1); | |
580 | xmemcpy(post_buf, t, post_buf_sz); | |
581 | *(post_buf + post_buf_sz) = '\0'; | |
30a4f2a8 | 582 | *t = '\0'; |
090089c4 | 583 | } |
584 | } | |
30a4f2a8 | 585 | if (buflen < DISK_PAGE_SIZE) { |
9864ee44 | 586 | buf = get_free_8k_page(); |
587 | memset(buf, '\0', buflen); | |
588 | buftype = BUF_TYPE_8K; | |
30a4f2a8 | 589 | } else { |
9864ee44 | 590 | buf = xcalloc(buflen, 1); |
591 | buftype = BUF_TYPE_MALLOC; | |
090089c4 | 592 | } |
090089c4 | 593 | |
30a4f2a8 | 594 | sprintf(buf, "%s %s HTTP/1.0\r\n", |
595 | Method, | |
596 | *req->urlpath ? req->urlpath : "/"); | |
090089c4 | 597 | len = strlen(buf); |
30a4f2a8 | 598 | if (httpState->req_hdr) { /* we have to parse the request header */ |
599 | xbuf = xstrdup(httpState->req_hdr); | |
090089c4 | 600 | for (t = strtok(xbuf, crlf); t; t = strtok(NULL, crlf)) { |
620da955 | 601 | if (strncasecmp(t, "Connection:", 11) == 0) |
af00901c | 602 | continue; |
f78dec53 | 603 | if (strncasecmp(t, "Host:", 5) == 0) |
604 | saw_host = 1; | |
090089c4 | 605 | if (len + (int) strlen(t) > buflen - 10) |
606 | continue; | |
607 | strcat(buf, t); | |
608 | strcat(buf, crlf); | |
609 | len += strlen(t) + 2; | |
610 | } | |
611 | xfree(xbuf); | |
090089c4 | 612 | } |
daddb143 | 613 | /* Add Forwarded: header */ |
2daae136 | 614 | ybuf = get_free_4k_page(); |
620da955 | 615 | if (entry->mem_obj) |
616 | cfd = entry->mem_obj->fd_of_first_client; | |
caebbe00 | 617 | if (cfd > -1 && opt_forwarded_for) { |
30a4f2a8 | 618 | sprintf(ybuf, "%s for %s\r\n", ForwardedBy, fd_table[cfd].ipaddr); |
caebbe00 | 619 | } else { |
620 | sprintf(ybuf, "%s\r\n", ForwardedBy); | |
d9766e35 | 621 | } |
daddb143 | 622 | strcat(buf, ybuf); |
623 | len += strlen(ybuf); | |
2daae136 | 624 | put_free_4k_page(ybuf); |
daddb143 | 625 | ybuf = NULL; |
626 | ||
620da955 | 627 | /* Add IMS header */ |
628 | if (entry->lastmod && req->method == METHOD_GET) { | |
629 | debug(11, 3, "httpSendRequest: Adding IMS: %s\r\n", | |
cf49df4b | 630 | mkrfc1123(entry->lastmod)); |
620da955 | 631 | ybuf = get_free_4k_page(); |
cf49df4b | 632 | sprintf(ybuf, "If-Modified-Since: %s\r\n", mkrfc1123(entry->lastmod)); |
620da955 | 633 | strcat(buf, ybuf); |
634 | len += strlen(ybuf); | |
635 | put_free_4k_page(ybuf); | |
636 | } | |
f78dec53 | 637 | /* Add Host: header */ |
638 | if (!saw_host) { | |
639 | ybuf = get_free_4k_page(); | |
640 | sprintf(ybuf, "Host: %s\r\n", req->host); | |
641 | strcat(buf, ybuf); | |
642 | len += strlen(ybuf); | |
643 | put_free_4k_page(ybuf); | |
644 | } | |
090089c4 | 645 | strcat(buf, crlf); |
646 | len += 2; | |
647 | if (post_buf) { | |
4768dbe0 | 648 | xmemcpy(buf + len, post_buf, post_buf_sz); |
649 | len += post_buf_sz; | |
090089c4 | 650 | xfree(post_buf); |
651 | } | |
67508012 | 652 | debug(11, 6, "httpSendRequest: FD %d:\n%s\n", fd, buf); |
30a4f2a8 | 653 | comm_write(fd, |
14e59844 | 654 | buf, |
655 | len, | |
656 | 30, | |
657 | httpSendComplete, | |
9864ee44 | 658 | httpState, |
659 | buftype == BUF_TYPE_8K ? put_free_8k_page : xfree); | |
090089c4 | 660 | } |
661 | ||
b8d8561b | 662 | int |
663 | proxyhttpStart(edge * e, char *url, StoreEntry * entry) | |
090089c4 | 664 | { |
f5558c95 | 665 | int sock; |
30a4f2a8 | 666 | HttpStateData *httpState = NULL; |
7111c86a | 667 | request_t *request = NULL; |
090089c4 | 668 | |
7111c86a | 669 | debug(11, 3, "proxyhttpStart: \"%s %s\"\n", |
efbb0a25 | 670 | RequestMethodStr[entry->method], url); |
019dd986 | 671 | debug(11, 10, "proxyhttpStart: HTTP request header:\n%s\n", |
22e4fa85 | 672 | entry->mem_obj->mime_hdr); |
090089c4 | 673 | |
30a4f2a8 | 674 | if (e->options & NEIGHBOR_PROXY_ONLY) |
090089c4 | 675 | storeStartDeleteBehind(entry); |
676 | ||
677 | /* Create socket. */ | |
16b204c4 | 678 | sock = comm_open(SOCK_STREAM, |
679 | 0, | |
680 | Config.Addrs.tcp_outgoing, | |
681 | 0, | |
682 | COMM_NONBLOCKING, | |
683 | url); | |
090089c4 | 684 | if (sock == COMM_ERROR) { |
019dd986 | 685 | debug(11, 4, "proxyhttpStart: Failed because we're out of sockets.\n"); |
b8de7ebe | 686 | squid_error_entry(entry, ERR_NO_FDS, xstrerror()); |
090089c4 | 687 | return COMM_ERROR; |
688 | } | |
30a4f2a8 | 689 | httpState = xcalloc(1, sizeof(HttpStateData)); |
690 | storeLockObject(httpState->entry = entry, NULL, NULL); | |
691 | httpState->req_hdr = entry->mem_obj->mime_hdr; | |
4768dbe0 | 692 | httpState->req_hdr_sz = entry->mem_obj->mime_hdr_sz; |
30a4f2a8 | 693 | request = get_free_request_t(); |
694 | httpState->request = requestLink(request); | |
695 | httpState->neighbor = e; | |
0d4d4170 | 696 | /* register the handler to free HTTP state data when the FD closes */ |
30a4f2a8 | 697 | comm_add_close_handler(sock, |
b177367b | 698 | httpStateFree, |
30a4f2a8 | 699 | (void *) httpState); |
784213dc | 700 | request->method = entry->method; |
701 | strncpy(request->host, e->host, SQUIDHOSTNAMELEN); | |
30a4f2a8 | 702 | request->port = e->http_port; |
784213dc | 703 | strncpy(request->urlpath, url, MAX_URL); |
a13d9042 | 704 | ipcache_nbgethostbyname(request->host, |
705 | sock, | |
b15fe823 | 706 | httpConnect, |
a13d9042 | 707 | httpState); |
708 | return COMM_OK; | |
709 | } | |
710 | ||
b15fe823 | 711 | static void |
e5f6c5c2 | 712 | httpConnect(int fd, ipcache_addrs * ia, void *data) |
a13d9042 | 713 | { |
714 | HttpStateData *httpState = data; | |
715 | request_t *request = httpState->request; | |
716 | StoreEntry *entry = httpState->entry; | |
e5f6c5c2 | 717 | if (ia == NULL) { |
a13d9042 | 718 | debug(11, 4, "httpConnect: Unknown host: %s\n", request->host); |
b8de7ebe | 719 | squid_error_entry(entry, ERR_DNS_FAIL, dns_error_message); |
a13d9042 | 720 | comm_close(fd); |
b15fe823 | 721 | return; |
090089c4 | 722 | } |
723 | /* Open connection. */ | |
e5f6c5c2 | 724 | httpState->connectState.fd = fd; |
725 | httpState->connectState.host = request->host; | |
726 | httpState->connectState.port = request->port; | |
727 | httpState->connectState.handler = httpConnectDone; | |
728 | httpState->connectState.data = httpState; | |
729 | comm_nbconnect(fd, &httpState->connectState); | |
730 | } | |
731 | ||
732 | static void | |
733 | httpConnectDone(int fd, int status, void *data) | |
734 | { | |
735 | HttpStateData *httpState = data; | |
736 | request_t *request = httpState->request; | |
737 | StoreEntry *entry = httpState->entry; | |
738 | edge *e = NULL; | |
739 | if (status != COMM_OK) { | |
740 | if ((e = httpState->neighbor)) { | |
741 | e->last_fail_time = squid_curtime; | |
742 | e->neighbor_up = 0; | |
090089c4 | 743 | } |
e5f6c5c2 | 744 | squid_error_entry(entry, ERR_CONNECT_FAIL, xstrerror()); |
745 | comm_close(fd); | |
746 | } else { | |
747 | /* Install connection complete handler. */ | |
748 | if (opt_no_ipcache) | |
749 | ipcacheInvalidate(request->host); | |
750 | fd_note(fd, entry->url); | |
b177367b | 751 | commSetSelect(fd, COMM_SELECT_LIFETIME, |
752 | httpLifetimeExpire, (void *) httpState, 0); | |
753 | commSetSelect(fd, COMM_SELECT_WRITE, | |
754 | httpSendRequest, (void *) httpState, 0); | |
090089c4 | 755 | } |
090089c4 | 756 | } |
757 | ||
b8d8561b | 758 | int |
4768dbe0 | 759 | httpStart(int unusedfd, |
760 | char *url, | |
761 | request_t * request, | |
762 | char *req_hdr, | |
763 | int req_hdr_sz, | |
764 | StoreEntry * entry) | |
090089c4 | 765 | { |
766 | /* Create state structure. */ | |
a13d9042 | 767 | int sock; |
30a4f2a8 | 768 | HttpStateData *httpState = NULL; |
090089c4 | 769 | |
7111c86a | 770 | debug(11, 3, "httpStart: \"%s %s\"\n", |
771 | RequestMethodStr[request->method], url); | |
019dd986 | 772 | debug(11, 10, "httpStart: req_hdr '%s'\n", req_hdr); |
090089c4 | 773 | |
090089c4 | 774 | /* Create socket. */ |
16b204c4 | 775 | sock = comm_open(SOCK_STREAM, |
776 | 0, | |
777 | Config.Addrs.tcp_outgoing, | |
778 | 0, | |
779 | COMM_NONBLOCKING, | |
780 | url); | |
090089c4 | 781 | if (sock == COMM_ERROR) { |
019dd986 | 782 | debug(11, 4, "httpStart: Failed because we're out of sockets.\n"); |
b8de7ebe | 783 | squid_error_entry(entry, ERR_NO_FDS, xstrerror()); |
090089c4 | 784 | return COMM_ERROR; |
785 | } | |
30a4f2a8 | 786 | httpState = xcalloc(1, sizeof(HttpStateData)); |
787 | storeLockObject(httpState->entry = entry, NULL, NULL); | |
788 | httpState->req_hdr = req_hdr; | |
4768dbe0 | 789 | httpState->req_hdr_sz = req_hdr_sz; |
30a4f2a8 | 790 | httpState->request = requestLink(request); |
791 | comm_add_close_handler(sock, | |
b177367b | 792 | httpStateFree, |
30a4f2a8 | 793 | (void *) httpState); |
a13d9042 | 794 | ipcache_nbgethostbyname(request->host, |
795 | sock, | |
796 | httpConnect, | |
797 | httpState); | |
090089c4 | 798 | return COMM_OK; |
799 | } | |
30a4f2a8 | 800 | |
b8d8561b | 801 | void |
802 | httpReplyHeaderStats(StoreEntry * entry) | |
30a4f2a8 | 803 | { |
804 | storeAppendPrintf(entry, open_bracket); | |
805 | storeAppendPrintf(entry, "{HTTP Reply Headers}\n"); | |
806 | storeAppendPrintf(entry, "{Headers parsed: %d}\n", | |
807 | ReplyHeaderStats.parsed); | |
808 | storeAppendPrintf(entry, "{ Date: %d}\n", | |
809 | ReplyHeaderStats.date); | |
810 | storeAppendPrintf(entry, "{ Last-Modified: %d}\n", | |
811 | ReplyHeaderStats.lm); | |
812 | storeAppendPrintf(entry, "{ Expires: %d}\n", | |
813 | ReplyHeaderStats.exp); | |
814 | storeAppendPrintf(entry, "{ Content-Type: %d}\n", | |
815 | ReplyHeaderStats.ctype); | |
816 | storeAppendPrintf(entry, "{Content-Length: %d}\n", | |
817 | ReplyHeaderStats.clen); | |
818 | storeAppendPrintf(entry, close_bracket); | |
819 | } |