[thirdparty/squid.git] / src / http.cc

/* $Id: http.cc,v 1.7 1996/03/25 19:05:50 wessels Exp $ */

#include "config.h"
#include <sys/errno.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "ansihelp.h"
#include "comm.h"
#include "store.h"
#include "stat.h"
#include "url.h"
#include "ipcache.h"
#include "cache_cf.h"
#include "ttl.h"
#include "icp.h"
#include "util.h"
#include "cached_error.h"

#define HTTP_PORT         80
#define HTTP_DELETE_GAP   (64*1024)
#define READBUFSIZ	4096

extern int errno;
extern char *dns_error_message;
extern time_t cached_curtime;

typedef struct _httpdata {
    StoreEntry *entry;
    char host[SQUIDHOSTNAMELEN + 1];
    int port;
    char *type;
    char *mime_hdr;
    char type_id;
    char request[MAX_URL + 1];
    char *icp_page_ptr;		/* Used to send proxy-http request: 
				 * put_free_8k_page(me) if the lifetime
				 * expires */
    char *icp_rwd_ptr;		/* When a lifetime expires during the
				 * middle of an icpwrite, don't lose the
				 * icpReadWriteData */
} HttpData;

extern char *tmp_error_buf;

char *HTTP_OPS[] =
{"GET", "POST", "HEAD", ""};

int http_url_parser(url, host, port, request)
     char *url;
     char *host;
     int *port;
     char *request;
{
    static char hostbuf[MAX_URL];
    static char atypebuf[MAX_URL];
    int t;

    /* initialize everything */
    (*port) = 0;
    atypebuf[0] = hostbuf[0] = request[0] = host[0] = '\0';

    t = sscanf(url, "%[a-zA-Z]://%[^/]%s", atypebuf, hostbuf, request);
    if ((t < 2) || (strcasecmp(atypebuf, "http") != 0)) {
	return -1;
    } else if (t == 2) {
	strcpy(request, "/");
    }
    if (sscanf(hostbuf, "%[^:]:%d", host, port) < 2)
	(*port) = HTTP_PORT;
    return 0;
}

int httpCachable(url, type, mime_hdr)
     char *url;
     char *type;
     char *mime_hdr;
{
    stoplist *p;

    /* GET and HEAD are cachable. Others are not. */
    if (((strncasecmp(type, "GET", 3) != 0)) &&
	(strncasecmp(type, "HEAD", 4) != 0))
	return 0;

    /* url's requiring authentication are uncachable */
    if (mime_hdr && (strstr(mime_hdr, "Authorization")))
	return 0;

    /* scan stop list */
    p = http_stoplist;
    while (p) {
	if (strstr(url, p->key))
	    return 0;
	p = p->next;
    }

    /* else cachable */
    return 1;
}

/* This will be called when timeout on read. */
void httpReadReplyTimeout(fd, data)
     int fd;
     HttpData *data;
{
    StoreEntry *entry = NULL;

    entry = data->entry;
    debug(4, "httpReadReplyTimeout: FD %d: <URL:%s>\n", fd, entry->url);
    cached_error_entry(entry, ERR_READ_TIMEOUT, NULL);
    if (data->icp_rwd_ptr)
	safe_free(data->icp_rwd_ptr);
    if (data->icp_page_ptr) {
	put_free_8k_page(data->icp_page_ptr);
	data->icp_page_ptr = NULL;
    }
    comm_set_select_handler(fd, COMM_SELECT_READ, 0, 0);
    comm_close(fd);
    safe_free(data);
}

/* This will be called when socket lifetime is expired. */
void httpLifetimeExpire(fd, data)
     int fd;
     HttpData *data;
{
    StoreEntry *entry = NULL;

    entry = data->entry;
    debug(4, "httpLifeTimeExpire: FD %d: <URL:%s>\n", fd, entry->url);

    cached_error_entry(entry, ERR_LIFETIME_EXP, NULL);
    if (data->icp_page_ptr) {
	put_free_8k_page(data->icp_page_ptr);
	data->icp_page_ptr = NULL;
    }
    if (data->icp_rwd_ptr)
	safe_free(data->icp_rwd_ptr);
    comm_set_select_handler(fd, COMM_SELECT_READ | COMM_SELECT_WRITE, 0, 0);
    comm_close(fd);
    safe_free(data);
}


/* This will be called when data is ready to be read from fd.  Read until
 * error or connection closed. */
void httpReadReply(fd, data)
     int fd;
     HttpData *data;
{
    static char buf[READBUFSIZ];
    int len;
    int clen;
    int off;
    StoreEntry *entry = NULL;

    entry = data->entry;
    if (entry->flag & DELETE_BEHIND) {
	if (storeClientWaiting(entry)) {
	    /* check if we want to defer reading */
	    clen = entry->mem_obj->e_current_len;
	    off = entry->mem_obj->e_lowest_offset;
	    if ((clen - off) > HTTP_DELETE_GAP) {
		debug(3, "httpReadReply: Read deferred for Object: %s\n",
		    entry->key);
		debug(3, "                Current Gap: %d bytes\n",
		    clen - off);

		/* reschedule, so it will be automatically reactivated
		 * when Gap is big enough. */
		comm_set_select_handler(fd,
		    COMM_SELECT_READ,
		    (PF) httpReadReply,
		    (caddr_t) data);

/* don't install read timeout until we are below the GAP */
#ifdef INSTALL_READ_TIMEOUT_ABOVE_GAP
		comm_set_select_handler_plus_timeout(fd,
		    COMM_SELECT_TIMEOUT,
		    (PF) httpReadReplyTimeout,
		    (caddr_t) data,
		    getReadTimeout());
#else
		comm_set_select_handler_plus_timeout(fd,
		    COMM_SELECT_TIMEOUT,
		    (PF) NULL,
		    (caddr_t) NULL,
		    (time_t) 0);
#endif
		return;
	    }
	} else {
	    /* we can terminate connection right now */
	    cached_error_entry(entry, ERR_NO_CLIENTS_BIG_OBJ, NULL);
	    comm_close(fd);
	    safe_free(data);
	    return;
	}
    }
    len = read(fd, buf, READBUFSIZ);
    debug(5, "httpReadReply: FD %d: len %d.\n", fd, len);

    if (len < 0 || ((len == 0) && (entry->mem_obj->e_current_len == 0))) {
	/* XXX we we should log when len==0 and current_len==0 */
	debug(2, "httpReadReply: FD %d: read failure: %s.\n",
	    fd, xstrerror());
	if (errno == ECONNRESET) {
	    /* Connection reset by peer */
	    /* consider it as a EOF */
	    if (!(entry->flag & DELETE_BEHIND))
		entry->expires = cached_curtime + ttlSet(entry);
	    sprintf(tmp_error_buf, "\n<p>Warning: The Remote Server sent RESET at the end of transmission.\n");
	    storeAppend(entry, tmp_error_buf, strlen(tmp_error_buf));
	    storeComplete(entry);
	} else {
	    cached_error_entry(entry, ERR_READ_ERROR, xstrerror());
	}
	comm_close(fd);
	safe_free(data);
    } else if (len == 0) {
	/* Connection closed; retrieval done. */
	if (!(entry->flag & DELETE_BEHIND))
	    entry->expires = cached_curtime + ttlSet(entry);
	storeComplete(entry);
	comm_close(fd);
	safe_free(data);
    } else if (((entry->mem_obj->e_current_len + len) > getHttpMax()) &&
	!(entry->flag & DELETE_BEHIND)) {
	/*  accept data, but start to delete behind it */
	storeStartDeleteBehind(entry);

	storeAppend(entry, buf, len);
	comm_set_select_handler(fd, COMM_SELECT_READ,
	    (PF) httpReadReply, (caddr_t) data);
	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	    (PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());

    } else if (entry->flag & CLIENT_ABORT_REQUEST) {
	/* append the last bit of info we get */
	storeAppend(entry, buf, len);
	cached_error_entry(entry, ERR_CLIENT_ABORT, NULL);
	comm_close(fd);
	safe_free(data);
    } else {
	storeAppend(entry, buf, len);
	comm_set_select_handler(fd, COMM_SELECT_READ,
	    (PF) httpReadReply, (caddr_t) data);
	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	    (PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());
    }
}

/* This will be called when request write is complete. Schedule read of
 * reply. */
void httpSendComplete(fd, buf, size, errflag, data)
     int fd;
     char *buf;
     int size;
     int errflag;
     HttpData *data;
{
    StoreEntry *entry = NULL;

    entry = data->entry;
    debug(5, "httpSendComplete: FD %d: size %d: errflag %d.\n",
	fd, size, errflag);

    if (buf) {
	put_free_8k_page(buf);	/* Allocated by httpSendRequest. */
	buf = NULL;
    }
    data->icp_page_ptr = NULL;	/* So lifetime expire doesn't re-free */
    data->icp_rwd_ptr = NULL;	/* Don't double free in lifetimeexpire */

    if (errflag) {
	cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
	comm_close(fd);
	safe_free(data);
	return;
    } else {
	/* Schedule read reply. */
	comm_set_select_handler(fd, COMM_SELECT_READ,
	    (PF) httpReadReply, (caddr_t) data);
	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	    (PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());
	comm_set_fd_lifetime(fd, -1);	/* disable lifetime DPW */

    }
}

/* This will be called when connect completes. Write request. */
void httpSendRequest(fd, data)
     int fd;
     HttpData *data;
{
    char *xbuf = NULL;
    char *ybuf = NULL;
    char *buf = NULL;
    char *t = NULL;
    char *post_buf = NULL;
    static char *crlf = "\r\n";
    static char *HARVEST_PROXY_TEXT = "via Harvest Cache version";
    int len = 0;
    int buflen;

    debug(5, "httpSendRequest: FD %d: data %p.\n", fd, data);
    buflen = strlen(data->type) + strlen(data->request);
    if (data->mime_hdr)
	buflen += strlen(data->mime_hdr);
    buflen += 512;		/* lots of extra */

    if (!strcasecmp(data->type, "POST") && data->mime_hdr) {
	if ((t = strstr(data->mime_hdr, "\r\n\r\n"))) {
	    post_buf = xstrdup(t + 4);
	    *(t + 4) = '\0';
	}
    }
    /* Since we limit the URL read to a 4K page, I doubt that the
     * mime header could be longer than an 8K page */
    buf = (char *) get_free_8k_page();
    data->icp_page_ptr = buf;
    if (buflen > DISK_PAGE_SIZE) {
	debug(0, "Mime header length %d is breaking ICP code\n", buflen);
    }
    memset(buf, '\0', buflen);

    sprintf(buf, "%s %s ", data->type, data->request);
    len = strlen(buf);
    if (data->mime_hdr) {	/* we have to parse the MIME header */
	xbuf = xstrdup(data->mime_hdr);
	for (t = strtok(xbuf, crlf); t; t = strtok(NULL, crlf)) {
	    if (strncasecmp(t, "User-Agent:", 11) == 0) {
		ybuf = (char *) get_free_4k_page();
		memset(ybuf, '\0', SM_PAGE_SIZE);
		sprintf(ybuf, "%s %s %s", t, HARVEST_PROXY_TEXT, SQUID_VERSION);
		t = ybuf;
	    }
	    if (strncasecmp(t, "If-Modified-Since:", 18) == 0)
		continue;
	    if (len + (int) strlen(t) > buflen - 10)
		continue;
	    strcat(buf, t);
	    strcat(buf, crlf);
	    len += strlen(t) + 2;
	}
	xfree(xbuf);
	if (ybuf) {
	    put_free_4k_page(ybuf);
	    ybuf = NULL;
	}
    }
    strcat(buf, crlf);
    len += 2;
    if (post_buf) {
	strcat(buf, post_buf);
	len += strlen(post_buf);
	xfree(post_buf);
    }
    debug(6, "httpSendRequest: FD %d: buf '%s'\n", fd, buf);
    data->icp_rwd_ptr = icpWrite(fd, buf, len, 30, httpSendComplete, data);
}

void httpConnInProgress(fd, data)
     int fd;
     HttpData *data;
{
    StoreEntry *entry = data->entry;

    if (comm_connect(fd, data->host, data->port) != COMM_OK)
	switch (errno) {
	case EINPROGRESS:
	case EALREADY:
	    /* schedule this handler again */
	    comm_set_select_handler(fd,
		COMM_SELECT_WRITE,
		(PF) httpConnInProgress,
		(caddr_t) data);
	    return;
	case EISCONN:
	    break;		/* cool, we're connected */
	default:
	    comm_close(fd);
	    cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
	    safe_free(data);
	    return;
	}
    /* Call the real write handler, now that we're fully connected */
    comm_set_select_handler(fd, COMM_SELECT_WRITE,
	(PF) httpSendRequest, (caddr_t) data);
}

int proxyhttpStart(e, url, entry)
     edge *e;
     char *url;
     StoreEntry *entry;
{

    /* Create state structure. */
    int sock, status;
    HttpData *data = (HttpData *) xmalloc(sizeof(HttpData));

    debug(3, "proxyhttpStart: <URL:%s>\n", url);
    debug(10, "proxyhttpStart: HTTP request header:\n%s\n",
	entry->mem_obj->mime_hdr);

    memset(data, '\0', sizeof(HttpData));
    data->entry = entry;

    strncpy(data->request, url, sizeof(data->request) - 1);
    data->type = HTTP_OPS[entry->type_id];
    data->port = e->ascii_port;
    data->mime_hdr = entry->mem_obj->mime_hdr;
    strncpy(data->host, e->host, sizeof(data->host) - 1);

    if (e->proxy_only)
	storeStartDeleteBehind(entry);

    /* Create socket. */
    sock = comm_open(COMM_NONBLOCKING, 0, 0, url);
    if (sock == COMM_ERROR) {
	debug(4, "proxyhttpStart: Failed because we're out of sockets.\n");
	cached_error_entry(entry, ERR_NO_FDS, xstrerror());
	safe_free(data);
	return COMM_ERROR;
    }
    /* check if IP is already in cache. It must be. 
     * It should be done before this route is called. 
     * Otherwise, we cannot check return code for connect. */
    if (!ipcache_gethostbyname(data->host)) {
	debug(4, "proxyhttpstart: Called without IP entry in ipcache. OR lookup failed.\n");
	comm_close(sock);
	cached_error_entry(entry, ERR_DNS_FAIL, dns_error_message);
	safe_free(data);
	return COMM_ERROR;
    }
    /* Open connection. */
    if ((status = comm_connect(sock, data->host, data->port))) {
	if (status != EINPROGRESS) {
	    comm_close(sock);
	    cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
	    safe_free(data);
	    e->last_fail_time = cached_curtime;
	    e->neighbor_up = 0;
	    return COMM_ERROR;
	} else {
	    debug(5, "proxyhttpStart: FD %d: EINPROGRESS.\n", sock);
	    comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
		(PF) httpLifetimeExpire, (caddr_t) data);
	    comm_set_select_handler(sock, COMM_SELECT_WRITE,
		(PF) httpConnInProgress, (caddr_t) data);
	    return COMM_OK;
	}
    }
    /* Install connection complete handler. */
    fd_note(sock, entry->url);
    comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	(PF) httpLifetimeExpire, (caddr_t) data);
    comm_set_select_handler(sock, COMM_SELECT_WRITE,
	(PF) httpSendRequest, (caddr_t) data);
    return COMM_OK;

}

int httpStart(unusedfd, url, type, mime_hdr, entry)
     int unusedfd;
     char *url;
     char *type;
     char *mime_hdr;
     StoreEntry *entry;
{
    /* Create state structure. */
    int sock, status;
    HttpData *data = (HttpData *) xmalloc(sizeof(HttpData));

    debug(3, "httpStart: %s <URL:%s>\n", type, url);
    debug(10, "httpStart: mime_hdr '%s'\n", mime_hdr);

    memset(data, '\0', sizeof(HttpData));
    data->entry = entry;
    data->type = type;
    data->mime_hdr = mime_hdr;

    /* Parse url. */
    if (http_url_parser(url, data->host, &data->port, data->request)) {
	cached_error_entry(entry, ERR_INVALID_URL, NULL);
	safe_free(data);
	return COMM_ERROR;
    }
    /* Create socket. */
    sock = comm_open(COMM_NONBLOCKING, 0, 0, url);
    if (sock == COMM_ERROR) {
	debug(4, "httpStart: Failed because we're out of sockets.\n");
	cached_error_entry(entry, ERR_NO_FDS, xstrerror());
	safe_free(data);
	return COMM_ERROR;
    }
    /* check if IP is already in cache. It must be. 
     * It should be done before this route is called. 
     * Otherwise, we cannot check return code for connect. */
    if (!ipcache_gethostbyname(data->host)) {
	debug(4, "httpstart: Called without IP entry in ipcache. OR lookup failed.\n");
	comm_close(sock);
	cached_error_entry(entry, ERR_DNS_FAIL, dns_error_message);
	safe_free(data);
	return COMM_ERROR;
    }
    /* Open connection. */
    if ((status = comm_connect(sock, data->host, data->port))) {
	if (status != EINPROGRESS) {
	    comm_close(sock);
	    cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
	    safe_free(data);
	    return COMM_ERROR;
	} else {
	    debug(5, "httpStart: FD %d: EINPROGRESS.\n", sock);
	    comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
		(PF) httpLifetimeExpire, (caddr_t) data);
	    comm_set_select_handler(sock, COMM_SELECT_WRITE,
		(PF) httpConnInProgress, (caddr_t) data);
	    return COMM_OK;
	}
    }
    /* Install connection complete handler. */
    fd_note(sock, entry->url);
    comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	(PF) httpLifetimeExpire, (caddr_t) data);
    comm_set_select_handler(sock, COMM_SELECT_WRITE,
	(PF) httpSendRequest, (caddr_t) data);
    return COMM_OK;
}
Commit	Line	Data
b367f261	1	/* $Id: http.cc,v 1.7 1996/03/25 19:05:50 wessels Exp $ */
ed43818f	2
090089c4	3	#include "config.h"
	4	#include <sys/errno.h>
	5	#include <stdlib.h>
	6	#include <string.h>
	7	#include <unistd.h>
	8
	9	#include "ansihelp.h"
	10	#include "comm.h"
	11	#include "store.h"
	12	#include "stat.h"
	13	#include "url.h"
	14	#include "ipcache.h"
	15	#include "cache_cf.h"
	16	#include "ttl.h"
	17	#include "icp.h"
	18	#include "util.h"
d1cfbef7	19	#include "cached_error.h"
090089c4	20
	21	#define HTTP_PORT 80
	22	#define HTTP_DELETE_GAP (64*1024)
b367f261	23	#define READBUFSIZ 4096
090089c4	24
	25	extern int errno;
	26	extern char *dns_error_message;
	27	extern time_t cached_curtime;
	28
	29	typedef struct _httpdata {
	30	StoreEntry *entry;
ed43818f	31	char host[SQUIDHOSTNAMELEN + 1];
090089c4	32	int port;
	33	char *type;
	34	char *mime_hdr;
	35	char type_id;
	36	char request[MAX_URL + 1];
	37	char icp_page_ptr; / Used to send proxy-http request:
	38	* put_free_8k_page(me) if the lifetime
	39	* expires */
	40	char icp_rwd_ptr; / When a lifetime expires during the
	41	* middle of an icpwrite, don't lose the
	42	* icpReadWriteData */
	43	} HttpData;
	44
	45	extern char *tmp_error_buf;
	46
	47	char *HTTP_OPS[] =
	48	{"GET", "POST", "HEAD", ""};
	49
	50	int http_url_parser(url, host, port, request)
	51	char *url;
	52	char *host;
	53	int *port;
	54	char *request;
	55	{
	56	static char hostbuf[MAX_URL];
	57	static char atypebuf[MAX_URL];
	58	int t;
	59
	60	/* initialize everything */
	61	(*port) = 0;
	62	atypebuf[0] = hostbuf[0] = request[0] = host[0] = '\0';
	63
	64	t = sscanf(url, "%[a-zA-Z]://%[^/]%s", atypebuf, hostbuf, request);
	65	if ((t < 2) \|\| (strcasecmp(atypebuf, "http") != 0)) {
	66	return -1;
	67	} else if (t == 2) {
	68	strcpy(request, "/");
	69	}
	70	if (sscanf(hostbuf, "%[^:]:%d", host, port) < 2)
	71	(*port) = HTTP_PORT;
	72	return 0;
	73	}
	74
	75	int httpCachable(url, type, mime_hdr)
	76	char *url;
	77	char *type;
	78	char *mime_hdr;
	79	{
	80	stoplist *p;
	81
	82	/* GET and HEAD are cachable. Others are not. */
	83	if (((strncasecmp(type, "GET", 3) != 0)) &&
	84	(strncasecmp(type, "HEAD", 4) != 0))
	85	return 0;
	86
	87	/* url's requiring authentication are uncachable */
	88	if (mime_hdr && (strstr(mime_hdr, "Authorization")))
	89	return 0;
	90
	91	/* scan stop list */
	92	p = http_stoplist;
	93	while (p) {
	94	if (strstr(url, p->key))
	95	return 0;
96	p = p->next;
97	}
98
99	/* else cachable */
100	return 1;
101	}
102
103	/* This will be called when timeout on read. */
104	void httpReadReplyTimeout(fd, data)
105	int fd;
106	HttpData *data;
107	{
108	StoreEntry *entry = NULL;
109
110	entry = data->entry;
111	debug(4, "httpReadReplyTimeout: FD %d: <URL:%s>\n", fd, entry->url);
b367f261	112	cached_error_entry(entry, ERR_READ_TIMEOUT, NULL);
090089c4	113	if (data->icp_rwd_ptr)
	114	safe_free(data->icp_rwd_ptr);
	115	if (data->icp_page_ptr) {
	116	put_free_8k_page(data->icp_page_ptr);
	117	data->icp_page_ptr = NULL;
	118	}
090089c4	119	comm_set_select_handler(fd, COMM_SELECT_READ, 0, 0);
090089c4	120	comm_close(fd);
090089c4	121	safe_free(data);
	122	}
	123
	124	/* This will be called when socket lifetime is expired. */
	125	void httpLifetimeExpire(fd, data)
	126	int fd;
	127	HttpData *data;
	128	{
	129	StoreEntry *entry = NULL;
	130
	131	entry = data->entry;
	132	debug(4, "httpLifeTimeExpire: FD %d: <URL:%s>\n", fd, entry->url);
	133
b367f261	134	cached_error_entry(entry, ERR_LIFETIME_EXP, NULL);
090089c4	135	if (data->icp_page_ptr) {
	136	put_free_8k_page(data->icp_page_ptr);
	137	data->icp_page_ptr = NULL;
	138	}
	139	if (data->icp_rwd_ptr)
	140	safe_free(data->icp_rwd_ptr);
090089c4	141	comm_set_select_handler(fd, COMM_SELECT_READ \| COMM_SELECT_WRITE, 0, 0);
090089c4	142	comm_close(fd);
090089c4	143	safe_free(data);
	144	}
	145
	146
	147
	148	/* This will be called when data is ready to be read from fd. Read until
	149	* error or connection closed. */
	150	void httpReadReply(fd, data)
	151	int fd;
	152	HttpData *data;
	153	{
b367f261	154	static char buf[READBUFSIZ];
090089c4	155	int len;
	156	int clen;
	157	int off;
	158	StoreEntry *entry = NULL;
	159
	160	entry = data->entry;
	161	if (entry->flag & DELETE_BEHIND) {
	162	if (storeClientWaiting(entry)) {
	163	/* check if we want to defer reading */
22e4fa85	164	clen = entry->mem_obj->e_current_len;
22e4fa85	165	off = entry->mem_obj->e_lowest_offset;
090089c4	166	if ((clen - off) > HTTP_DELETE_GAP) {
	167	debug(3, "httpReadReply: Read deferred for Object: %s\n",
	168	entry->key);
	169	debug(3, " Current Gap: %d bytes\n",
	170	clen - off);
	171
	172	/* reschedule, so it will be automatically reactivated
	173	* when Gap is big enough. */
	174	comm_set_select_handler(fd,
	175	COMM_SELECT_READ,
	176	(PF) httpReadReply,
	177	(caddr_t) data);
	178
	179	/* don't install read timeout until we are below the GAP */
	180	#ifdef INSTALL_READ_TIMEOUT_ABOVE_GAP
	181	comm_set_select_handler_plus_timeout(fd,
	182	COMM_SELECT_TIMEOUT,
	183	(PF) httpReadReplyTimeout,
	184	(caddr_t) data,
	185	getReadTimeout());
	186	#else
	187	comm_set_select_handler_plus_timeout(fd,
	188	COMM_SELECT_TIMEOUT,
	189	(PF) NULL,
	190	(caddr_t) NULL,
	191	(time_t) 0);
	192	#endif
	193	return;
	194	}
	195	} else {
	196	/* we can terminate connection right now */
b367f261	197	cached_error_entry(entry, ERR_NO_CLIENTS_BIG_OBJ, NULL);
090089c4	198	comm_close(fd);
090089c4	199	safe_free(data);
	200	return;
	201	}
	202	}
b367f261	203	len = read(fd, buf, READBUFSIZ);
090089c4	204	debug(5, "httpReadReply: FD %d: len %d.\n", fd, len);
090089c4	205
22e4fa85	206	if (len < 0 \|\| ((len == 0) && (entry->mem_obj->e_current_len == 0))) {
090089c4	207	/* XXX we we should log when len==0 and current_len==0 */
	208	debug(2, "httpReadReply: FD %d: read failure: %s.\n",
	209	fd, xstrerror());
	210	if (errno == ECONNRESET) {
	211	/* Connection reset by peer */
	212	/* consider it as a EOF */
	213	if (!(entry->flag & DELETE_BEHIND))
	214	entry->expires = cached_curtime + ttlSet(entry);
	215	sprintf(tmp_error_buf, "\n<p>Warning: The Remote Server sent RESET at the end of transmission.\n");
	216	storeAppend(entry, tmp_error_buf, strlen(tmp_error_buf));
	217	storeComplete(entry);
	218	} else {
b367f261	219	cached_error_entry(entry, ERR_READ_ERROR, xstrerror());
090089c4	220	}
090089c4	221	comm_close(fd);
090089c4	222	safe_free(data);
	223	} else if (len == 0) {
	224	/* Connection closed; retrieval done. */
	225	if (!(entry->flag & DELETE_BEHIND))
	226	entry->expires = cached_curtime + ttlSet(entry);
	227	storeComplete(entry);
	228	comm_close(fd);
	229	safe_free(data);
22e4fa85	230	} else if (((entry->mem_obj->e_current_len + len) > getHttpMax()) &&
090089c4	231	!(entry->flag & DELETE_BEHIND)) {
	232	/* accept data, but start to delete behind it */
	233	storeStartDeleteBehind(entry);
	234
	235	storeAppend(entry, buf, len);
	236	comm_set_select_handler(fd, COMM_SELECT_READ,
	237	(PF) httpReadReply, (caddr_t) data);
	238	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	239	(PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());
	240
	241	} else if (entry->flag & CLIENT_ABORT_REQUEST) {
	242	/* append the last bit of info we get */
	243	storeAppend(entry, buf, len);
b367f261	244	cached_error_entry(entry, ERR_CLIENT_ABORT, NULL);
090089c4	245	comm_close(fd);
090089c4	246	safe_free(data);
	247	} else {
	248	storeAppend(entry, buf, len);
	249	comm_set_select_handler(fd, COMM_SELECT_READ,
	250	(PF) httpReadReply, (caddr_t) data);
	251	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	252	(PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());
	253	}
	254	}
	255
	256	/* This will be called when request write is complete. Schedule read of
	257	* reply. */
	258	void httpSendComplete(fd, buf, size, errflag, data)
	259	int fd;
	260	char *buf;
	261	int size;
	262	int errflag;
	263	HttpData *data;
	264	{
	265	StoreEntry *entry = NULL;
	266
	267	entry = data->entry;
	268	debug(5, "httpSendComplete: FD %d: size %d: errflag %d.\n",
	269	fd, size, errflag);
	270
	271	if (buf) {
	272	put_free_8k_page(buf); /* Allocated by httpSendRequest. */
	273	buf = NULL;
	274	}
	275	data->icp_page_ptr = NULL; /* So lifetime expire doesn't re-free */
	276	data->icp_rwd_ptr = NULL; /* Don't double free in lifetimeexpire */
	277
	278	if (errflag) {
b367f261	279	cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
090089c4	280	comm_close(fd);
090089c4	281	safe_free(data);
	282	return;
	283	} else {
	284	/* Schedule read reply. */
	285	comm_set_select_handler(fd, COMM_SELECT_READ,
	286	(PF) httpReadReply, (caddr_t) data);
	287	comm_set_select_handler_plus_timeout(fd, COMM_SELECT_TIMEOUT,
	288	(PF) httpReadReplyTimeout, (caddr_t) data, getReadTimeout());
	289	comm_set_fd_lifetime(fd, -1); /* disable lifetime DPW */
	290
	291	}
	292	}
	293
	294	/* This will be called when connect completes. Write request. */
	295	void httpSendRequest(fd, data)
	296	int fd;
	297	HttpData *data;
	298	{
	299	char *xbuf = NULL;
	300	char *ybuf = NULL;
	301	char *buf = NULL;
	302	char *t = NULL;
	303	char *post_buf = NULL;
	304	static char *crlf = "\r\n";
	305	static char *HARVEST_PROXY_TEXT = "via Harvest Cache version";
	306	int len = 0;
	307	int buflen;
	308
	309	debug(5, "httpSendRequest: FD %d: data %p.\n", fd, data);
	310	buflen = strlen(data->type) + strlen(data->request);
	311	if (data->mime_hdr)
	312	buflen += strlen(data->mime_hdr);
	313	buflen += 512; /* lots of extra */
	314
	315	if (!strcasecmp(data->type, "POST") && data->mime_hdr) {
	316	if ((t = strstr(data->mime_hdr, "\r\n\r\n"))) {
	317	post_buf = xstrdup(t + 4);
	318	*(t + 4) = '\0';
	319	}
	320	}
	321	/* Since we limit the URL read to a 4K page, I doubt that the
	322	* mime header could be longer than an 8K page */
	323	buf = (char *) get_free_8k_page();
	324	data->icp_page_ptr = buf;
	325	if (buflen > DISK_PAGE_SIZE) {
	326	debug(0, "Mime header length %d is breaking ICP code\n", buflen);
	327	}
	328	memset(buf, '\0', buflen);
	329
	330	sprintf(buf, "%s %s ", data->type, data->request);
	331	len = strlen(buf);
	332	if (data->mime_hdr) { /* we have to parse the MIME header */
	333	xbuf = xstrdup(data->mime_hdr);
	334	for (t = strtok(xbuf, crlf); t; t = strtok(NULL, crlf)) {
	335	if (strncasecmp(t, "User-Agent:", 11) == 0) {
	336	ybuf = (char *) get_free_4k_page();
	337	memset(ybuf, '\0', SM_PAGE_SIZE);
73a5465f	338	sprintf(ybuf, "%s %s %s", t, HARVEST_PROXY_TEXT, SQUID_VERSION);
090089c4	339	t = ybuf;
	340	}
	341	if (strncasecmp(t, "If-Modified-Since:", 18) == 0)
	342	continue;
	343	if (len + (int) strlen(t) > buflen - 10)
	344	continue;
	345	strcat(buf, t);
	346	strcat(buf, crlf);
	347	len += strlen(t) + 2;
	348	}
	349	xfree(xbuf);
	350	if (ybuf) {
	351	put_free_4k_page(ybuf);
	352	ybuf = NULL;
	353	}
	354	}
	355	strcat(buf, crlf);
	356	len += 2;
	357	if (post_buf) {
	358	strcat(buf, post_buf);
	359	len += strlen(post_buf);
	360	xfree(post_buf);
	361	}
	362	debug(6, "httpSendRequest: FD %d: buf '%s'\n", fd, buf);
	363	data->icp_rwd_ptr = icpWrite(fd, buf, len, 30, httpSendComplete, data);
	364	}
	365
	366	void httpConnInProgress(fd, data)
	367	int fd;
	368	HttpData *data;
	369	{
	370	StoreEntry *entry = data->entry;
	371
	372	if (comm_connect(fd, data->host, data->port) != COMM_OK)
	373	switch (errno) {
	374	case EINPROGRESS:
	375	case EALREADY:
	376	/* schedule this handler again */
	377	comm_set_select_handler(fd,
	378	COMM_SELECT_WRITE,
	379	(PF) httpConnInProgress,
	380	(caddr_t) data);
	381	return;
	382	case EISCONN:
	383	break; /* cool, we're connected */
	384	default:
	385	comm_close(fd);
b367f261	386	cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
090089c4	387	safe_free(data);
	388	return;
	389	}
	390	/* Call the real write handler, now that we're fully connected */
	391	comm_set_select_handler(fd, COMM_SELECT_WRITE,
	392	(PF) httpSendRequest, (caddr_t) data);
	393	}
	394
	395	int proxyhttpStart(e, url, entry)
	396	edge *e;
	397	char *url;
	398	StoreEntry *entry;
	399	{
	400
	401	/* Create state structure. */
	402	int sock, status;
	403	HttpData data = (HttpData ) xmalloc(sizeof(HttpData));
	404
	405	debug(3, "proxyhttpStart: <URL:%s>\n", url);
	406	debug(10, "proxyhttpStart: HTTP request header:\n%s\n",
22e4fa85	407	entry->mem_obj->mime_hdr);
090089c4	408
	409	memset(data, '\0', sizeof(HttpData));
	410	data->entry = entry;
	411
	412	strncpy(data->request, url, sizeof(data->request) - 1);
	413	data->type = HTTP_OPS[entry->type_id];
	414	data->port = e->ascii_port;
22e4fa85	415	data->mime_hdr = entry->mem_obj->mime_hdr;
090089c4	416	strncpy(data->host, e->host, sizeof(data->host) - 1);
	417
	418	if (e->proxy_only)
	419	storeStartDeleteBehind(entry);
	420
	421	/* Create socket. */
	422	sock = comm_open(COMM_NONBLOCKING, 0, 0, url);
	423	if (sock == COMM_ERROR) {
	424	debug(4, "proxyhttpStart: Failed because we're out of sockets.\n");
b367f261	425	cached_error_entry(entry, ERR_NO_FDS, xstrerror());
090089c4	426	safe_free(data);
	427	return COMM_ERROR;
	428	}
	429	/* check if IP is already in cache. It must be.
	430	* It should be done before this route is called.
	431	* Otherwise, we cannot check return code for connect. */
	432	if (!ipcache_gethostbyname(data->host)) {
	433	debug(4, "proxyhttpstart: Called without IP entry in ipcache. OR lookup failed.\n");
	434	comm_close(sock);
b367f261	435	cached_error_entry(entry, ERR_DNS_FAIL, dns_error_message);
090089c4	436	safe_free(data);
	437	return COMM_ERROR;
	438	}
	439	/* Open connection. */
	440	if ((status = comm_connect(sock, data->host, data->port))) {
	441	if (status != EINPROGRESS) {
	442	comm_close(sock);
b367f261	443	cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
090089c4	444	safe_free(data);
	445	e->last_fail_time = cached_curtime;
	446	e->neighbor_up = 0;
	447	return COMM_ERROR;
	448	} else {
	449	debug(5, "proxyhttpStart: FD %d: EINPROGRESS.\n", sock);
	450	comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	451	(PF) httpLifetimeExpire, (caddr_t) data);
	452	comm_set_select_handler(sock, COMM_SELECT_WRITE,
	453	(PF) httpConnInProgress, (caddr_t) data);
	454	return COMM_OK;
	455	}
	456	}
	457	/* Install connection complete handler. */
	458	fd_note(sock, entry->url);
	459	comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	460	(PF) httpLifetimeExpire, (caddr_t) data);
	461	comm_set_select_handler(sock, COMM_SELECT_WRITE,
	462	(PF) httpSendRequest, (caddr_t) data);
	463	return COMM_OK;
	464
	465	}
	466
	467	int httpStart(unusedfd, url, type, mime_hdr, entry)
	468	int unusedfd;
	469	char *url;
	470	char *type;
	471	char *mime_hdr;
	472	StoreEntry *entry;
	473	{
	474	/* Create state structure. */
	475	int sock, status;
	476	HttpData data = (HttpData ) xmalloc(sizeof(HttpData));
	477
	478	debug(3, "httpStart: %s <URL:%s>\n", type, url);
	479	debug(10, "httpStart: mime_hdr '%s'\n", mime_hdr);
	480
	481	memset(data, '\0', sizeof(HttpData));
	482	data->entry = entry;
	483	data->type = type;
	484	data->mime_hdr = mime_hdr;
	485
	486	/* Parse url. */
	487	if (http_url_parser(url, data->host, &data->port, data->request)) {
b367f261	488	cached_error_entry(entry, ERR_INVALID_URL, NULL);
090089c4	489	safe_free(data);
	490	return COMM_ERROR;
	491	}
	492	/* Create socket. */
	493	sock = comm_open(COMM_NONBLOCKING, 0, 0, url);
	494	if (sock == COMM_ERROR) {
	495	debug(4, "httpStart: Failed because we're out of sockets.\n");
b367f261	496	cached_error_entry(entry, ERR_NO_FDS, xstrerror());
090089c4	497	safe_free(data);
	498	return COMM_ERROR;
	499	}
	500	/* check if IP is already in cache. It must be.
	501	* It should be done before this route is called.
	502	* Otherwise, we cannot check return code for connect. */
	503	if (!ipcache_gethostbyname(data->host)) {
	504	debug(4, "httpstart: Called without IP entry in ipcache. OR lookup failed.\n");
	505	comm_close(sock);
b367f261	506	cached_error_entry(entry, ERR_DNS_FAIL, dns_error_message);
090089c4	507	safe_free(data);
	508	return COMM_ERROR;
	509	}
	510	/* Open connection. */
	511	if ((status = comm_connect(sock, data->host, data->port))) {
	512	if (status != EINPROGRESS) {
	513	comm_close(sock);
b367f261	514	cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
090089c4	515	safe_free(data);
	516	return COMM_ERROR;
	517	} else {
	518	debug(5, "httpStart: FD %d: EINPROGRESS.\n", sock);
	519	comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	520	(PF) httpLifetimeExpire, (caddr_t) data);
	521	comm_set_select_handler(sock, COMM_SELECT_WRITE,
	522	(PF) httpConnInProgress, (caddr_t) data);
	523	return COMM_OK;
	524	}
	525	}
	526	/* Install connection complete handler. */
	527	fd_note(sock, entry->url);
	528	comm_set_select_handler(sock, COMM_SELECT_LIFETIME,
	529	(PF) httpLifetimeExpire, (caddr_t) data);
	530	comm_set_select_handler(sock, COMM_SELECT_WRITE,
	531	(PF) httpSendRequest, (caddr_t) data);
	532	return COMM_OK;
	533	}