From: wessels <> Date: Mon, 5 May 1997 09:43:38 +0000 (+0000) Subject: - DO NOT set the filemap bits for unvalidated entries. Doing so X-Git-Tag: SQUID_3_0_PRE1~5026 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9e4ad609cdc6849a20b298e7c2437007109b80fc;p=thirdparty%2Fsquid.git - DO NOT set the filemap bits for unvalidated entries. Doing so causes lots of race-condition problems, especially when reading DIRTY logs. - Install icpDetectClientClose handler earlier. Move from access check to just after reading the request. - Log open FD's upon exit. - Added dnsShutdownRead handler to detect dnsserver shutdowns and free up state. - Added doubleAverage and integerAverage for moving average calculations. - misc prototype cleanup - misc debug_trap's and fatal_dump's - Removed some thread splitting for fooStartComplete() because the various protcol start routines should never need a swapin callback. The object should already be locked and swapped in. - queue_length counter for IP cache. - Make it okay to storeUnlock a pending object, so long as its not been DISPATCHED yet. - fixed hybrid store rebuild code. - fixed file_open() flags to TRUNC new swaplog's - Claned up wierd (FILE *) stuff in tools.c, always write to debug_log(). - rewrote writePidFile() to use file_open(). --- diff --git a/src/client_side.cc b/src/client_side.cc index 12dd8f1f5b..3b45c009be 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.100 1997/05/02 04:28:34 wessels Exp $ + * $Id: client_side.cc,v 1.101 1997/05/05 03:43:38 wessels Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -32,7 +32,7 @@ #include "squid.h" static RH clientRedirectDone; -static void icpHandleIMSReply _PARAMS((int fd, StoreEntry * entry, void *data)); +static PIF icpHandleIMSReply; static int clientGetsOldEntry _PARAMS((StoreEntry * new, StoreEntry * old, request_t * request)); static int checkAccelOnly _PARAMS((icpStateData * icpState)); @@ -189,13 +189,6 @@ clientRedirectDone(void *data, char *result) } icpParseRequestHeaders(icpState); fd_note(fd, icpState->url); - if (!BIT_TEST(icpState->request->flags, REQ_PROXY_KEEPALIVE)) { - commSetSelect(fd, - COMM_SELECT_READ, - icpDetectClientClose, - icpState, - 0); - } icpProcessRequest(fd, icpState); } diff --git a/src/comm.cc b/src/comm.cc index cc055e668f..a8a5b37bdc 100644 --- a/src/comm.cc +++ b/src/comm.cc @@ -1,6 +1,6 @@ /* - * $Id: comm.cc,v 1.149 1997/05/02 21:34:04 wessels Exp $ + * $Id: comm.cc,v 1.150 1997/05/05 03:43:38 wessels Exp $ * * DEBUG: section 5 Socket Functions * AUTHOR: Harvest Derived @@ -753,7 +753,7 @@ comm_select(time_t sec) if (shutdown_pending > 0 || reread_pending > 0) setSocketShutdownLifetimes(Config.shutdownLifetime); else - setSocketShutdownLifetimes(0); + setSocketShutdownLifetimes(1); } nfds = 0; maxfd = Biggest_FD + 1; @@ -1383,15 +1383,13 @@ commHandleWrite(int fd, void *data) /* Select for Writing on FD, until SIZE bytes are sent. Call * * HANDLER when complete. */ void -comm_write(int fd, char *buf, int size, CWCB * handler, void *handler_data, FREE *free_func) +comm_write(int fd, char *buf, int size, CWCB * handler, void *handler_data, FREE * free_func) { CommWriteStateData *state = NULL; debug(5, 5, "comm_write: FD %d: sz %d: hndl %p: data %p.\n", fd, size, handler, handler_data); - if (fd_table[fd].rwstate) { - debug(5, 1, "WARNING: FD %d: A comm_write is already active.\n", fd); - CommWriteStateCallbackAndFree(fd, COMM_ERROR); - } + if (fd_table[fd].rwstate) + fatal_dump("comm_write: comm_write is already active"); state = xcalloc(1, sizeof(CommWriteStateData)); state->buf = buf; state->size = size; diff --git a/src/disk.cc b/src/disk.cc index ab4805b525..d9e4b2f610 100644 --- a/src/disk.cc +++ b/src/disk.cc @@ -1,5 +1,5 @@ /* - * $Id: disk.cc,v 1.67 1997/05/02 21:34:06 wessels Exp $ + * $Id: disk.cc,v 1.68 1997/05/05 03:43:39 wessels Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -148,7 +148,7 @@ disk_init(void) /* Open a disk file. Return a file descriptor */ int -file_open(const char *path, int mode, FOCB *callback, void *callback_data) +file_open(const char *path, int mode, FOCB * callback, void *callback_data) { int fd; open_ctrl_t *ctrlp; @@ -165,8 +165,8 @@ file_open(const char *path, int mode, FOCB *callback, void *callback_data) /* Open file */ #if USE_ASYNC_IO if (callback != NULL) { - aioOpen(path, mode, 0644, file_open_complete, ctrlp); - return DISK_OK; + aioOpen(path, mode, 0644, file_open_complete, ctrlp); + return DISK_OK; } #endif fd = open(path, mode, 0644); @@ -387,7 +387,7 @@ file_write(int fd, int len, FILE_WRITE_HD handle, void *handle_data, - void (*free_func) _PARAMS((void *))) + FREE * free_func) { dwrite_q *wq = NULL; FD_ENTRY *fde; diff --git a/src/dns.cc b/src/dns.cc index bdb1bf2bba..b887976f91 100644 --- a/src/dns.cc +++ b/src/dns.cc @@ -1,5 +1,5 @@ /* - * $Id: dns.cc,v 1.33 1997/04/30 18:30:47 wessels Exp $ + * $Id: dns.cc,v 1.34 1997/05/05 03:43:40 wessels Exp $ * * DEBUG: section 34 Dnsserver interface * AUTHOR: Harvest Derived @@ -111,6 +111,7 @@ struct dnsQueueData { }; static int dnsOpenServer _PARAMS((const char *command)); +static PF dnsShutdownRead; static dnsserver_t **dns_child_table = NULL; @@ -370,6 +371,27 @@ dnsShutdownServers(void) NULL, /* Handler */ NULL, /* Handler-data */ xfree); + commSetSelect(dnsData->inpipe, + COMM_SELECT_READ, + dnsShutdownRead, + dnsData, + 0); dnsData->flags |= DNS_FLAG_CLOSING; } } + +static void +dnsShutdownRead(int fd, void *data) +{ + dnsserver_t *dnsData = data; + debug(14, dnsData->flags & DNS_FLAG_CLOSING ? 5 : 1, + "FD %d: Connection from DNSSERVER #%d is closed, disabling\n", + fd, + dnsData->id); + dnsData->flags = 0; + commSetSelect(fd, + COMM_SELECT_WRITE, + NULL, + NULL, 0); + comm_close(fd); +} diff --git a/src/fd.cc b/src/fd.cc index d4fa4f5bc2..f984c6fbfd 100644 --- a/src/fd.cc +++ b/src/fd.cc @@ -31,6 +31,12 @@ void fd_close(int fd) { FD_ENTRY *fde = &fd_table[fd]; + if (fde->type == FD_FILE) { + if (fde->read_handler) + fatal_dump("file_close: read_handler present"); + if (fde->write_handler) + fatal_dump("file_close: write_handler present"); + } fdUpdateBiggest(fd, fde->open = FD_CLOSE); memset(fde, '\0', sizeof(FD_ENTRY)); fde->timeout = 0; @@ -70,5 +76,20 @@ fd_bytes(int fd, int len, unsigned int type) void fdFreeMemory(void) { - safe_free(fd_table); + safe_free(fd_table); +} + +void +fdDumpOpen(void) +{ + int i; + FD_ENTRY *fde; + for (i = 0; i < Squid_MaxFD; i++) { + fde = &fd_table[i]; + if (!fde->open) + continue; + if (i == fileno(debug_log)) + continue; + debug(5, 1, "Open FD %4d %s\n", i, fde->desc); + } } diff --git a/src/fqdncache.cc b/src/fqdncache.cc index 66d9a7c192..383a04d973 100644 --- a/src/fqdncache.cc +++ b/src/fqdncache.cc @@ -1,6 +1,6 @@ /* - * $Id: fqdncache.cc,v 1.52 1997/05/02 04:28:34 wessels Exp $ + * $Id: fqdncache.cc,v 1.53 1997/05/05 03:43:42 wessels Exp $ * * DEBUG: section 35 FQDN Cache * AUTHOR: Harvest Derived @@ -526,7 +526,6 @@ fqdncache_dnsHandleRead(int fd, void *data) { dnsserver_t *dnsData = data; int len; - int svc_time; int n; fqdncache_entry *f = NULL; fqdncache_entry *x = NULL; @@ -566,11 +565,10 @@ fqdncache_dnsHandleRead(int fd, void *data) fatal_dump("fqdncache_dnsHandleRead: bad status"); if (strstr(dnsData->ip_inbuf, "$end\n")) { /* end of record found */ - svc_time = tvSubMsec(dnsData->dispatch_time, current_time); - if (n > FQDNCACHE_AV_FACTOR) - n = FQDNCACHE_AV_FACTOR; - FqdncacheStats.avg_svc_time - = (FqdncacheStats.avg_svc_time * (n - 1) + svc_time) / n; + FqdncacheStats.avg_svc_time = + intAverage(FqdncacheStats.avg_svc_time, + tvSubMsec(dnsData->dispatch_time, current_time), + n, FQDNCACHE_AV_FACTOR); if ((x = fqdncache_parsebuffer(dnsData->ip_inbuf, dnsData)) == NULL) { debug(35, 0, "fqdncache_dnsHandleRead: fqdncache_parsebuffer failed?!\n"); } else { diff --git a/src/ftp.cc b/src/ftp.cc index e8f167946f..d84957ea42 100644 --- a/src/ftp.cc +++ b/src/ftp.cc @@ -1,6 +1,6 @@ /* - * $Id: ftp.cc,v 1.103 1997/04/30 18:30:50 wessels Exp $ + * $Id: ftp.cc,v 1.104 1997/05/05 03:43:42 wessels Exp $ * * DEBUG: section 9 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived @@ -133,18 +133,18 @@ typedef struct ftp_ctrl_t { } ftp_ctrl_t; /* Local functions */ -static const char *ftpTransferMode _PARAMS((const char *)); -static char *ftpGetBasicAuth _PARAMS((const char *)); -static void ftpStateFree _PARAMS((int, void *)); -static void ftpReadReply _PARAMS((int, void *)); -static void ftpStartComplete _PARAMS((void *, int)); -static void ftpConnectDone _PARAMS((int fd, int status, void *data)); +static CNCB ftpConnectDone; +static CWCB ftpSendComplete; +static PF ftpReadReply; +static PF ftpSendRequest; +static PF ftpServerClosed; +static PF ftpStateFree; static PF ftpTimeout; +static char *ftpGetBasicAuth _PARAMS((const char *)); +static const char *ftpTransferMode _PARAMS((const char *)); static void ftpProcessReplyHeader _PARAMS((FtpStateData *, const char *, int)); -static void ftpSendComplete _PARAMS((int, char *, int, int, void *)); -static void ftpSendRequest _PARAMS((int, void *)); -static void ftpServerClosed _PARAMS((int, void *)); -static void ftp_login_parser _PARAMS((const char *, FtpStateData *)); +static void ftpStartComplete _PARAMS((void *, int)); +static void ftpLoginParser _PARAMS((const char *, FtpStateData *)); /* External functions */ extern char *base64_decode _PARAMS((const char *coded)); @@ -165,7 +165,7 @@ ftpStateFree(int fd, void *data) } static void -ftp_login_parser(const char *login, FtpStateData * ftpState) +ftpLoginParser(const char *login, FtpStateData * ftpState) { char *s = NULL; xstrncpy(ftpState->user, login, MAX_URL); @@ -539,7 +539,7 @@ ftpCheckAuth(FtpStateData * ftpState, char *req_hdr) { char *orig_user; char *auth; - ftp_login_parser(ftpState->request->login, ftpState); + ftpLoginParser(ftpState->request->login, ftpState); if (ftpState->user[0] && ftpState->password[0]) return 1; /* name and passwd both in URL */ if (!ftpState->user[0] && !ftpState->password[0]) @@ -550,7 +550,7 @@ ftpCheckAuth(FtpStateData * ftpState, char *req_hdr) if ((auth = ftpGetBasicAuth(req_hdr)) == NULL) return 0; /* need auth header */ orig_user = xstrdup(ftpState->user); - ftp_login_parser(auth, ftpState); + ftpLoginParser(auth, ftpState); if (!strcmp(orig_user, ftpState->user)) { xfree(orig_user); return 1; /* same username */ diff --git a/src/gopher.cc b/src/gopher.cc index adc8a47d01..9dc0365f0f 100644 --- a/src/gopher.cc +++ b/src/gopher.cc @@ -1,5 +1,5 @@ /* - * $Id: gopher.cc,v 1.77 1997/04/30 18:30:52 wessels Exp $ + * $Id: gopher.cc,v 1.78 1997/05/05 03:43:43 wessels Exp $ * * DEBUG: section 10 Gopher * AUTHOR: Harvest Derived @@ -912,7 +912,7 @@ gopherStartComplete(void *datap, int status) StoreEntry *entry = datap; char *url = entry->url; GopherStateData *gopherState = CreateGopherStateData(); - int sock; + int fd; gopherState->entry = entry; debug(10, 3, "gopherStart: url: %s\n", url); /* Parse url. */ @@ -923,19 +923,19 @@ gopherStartComplete(void *datap, int status) return; } /* Create socket. */ - sock = comm_open(SOCK_STREAM, + fd = comm_open(SOCK_STREAM, 0, Config.Addrs.tcp_outgoing, 0, COMM_NONBLOCKING, url); - if (sock == COMM_ERROR) { + if (fd == COMM_ERROR) { debug(10, 4, "gopherStart: Failed because we're out of sockets.\n"); squid_error_entry(entry, ERR_NO_FDS, xstrerror()); gopherStateFree(-1, gopherState); return; } - comm_add_close_handler(sock, + comm_add_close_handler(fd, gopherStateFree, gopherState); /* check if IP is already in cache. It must be. @@ -944,7 +944,7 @@ gopherStartComplete(void *datap, int status) if (!ipcache_gethostbyname(gopherState->host, 0)) { debug(10, 4, "gopherStart: Called without IP entry in ipcache. OR lookup failed.\n"); squid_error_entry(entry, ERR_DNS_FAIL, dns_error_message); - comm_close(sock); + comm_close(fd); return; } if (((gopherState->type_id == GOPHER_INDEX) || (gopherState->type_id == GOPHER_CSO)) @@ -964,11 +964,11 @@ gopherStartComplete(void *datap, int status) } gopherToHTML(gopherState, (char *) NULL, 0); storeComplete(entry); - comm_close(sock); + comm_close(fd); return; } - commSetTimeout(sock, Config.Timeout.connect, gopherTimeout, gopherState); - commConnectStart(sock, + commSetTimeout(fd, Config.Timeout.connect, gopherTimeout, gopherState); + commConnectStart(fd, gopherState->host, gopherState->port, gopherConnectDone, diff --git a/src/http.cc b/src/http.cc index 08bef47ad1..60bd4aa8cc 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,5 +1,5 @@ /* - * $Id: http.cc,v 1.158 1997/04/30 20:06:28 wessels Exp $ + * $Id: http.cc,v 1.159 1997/05/05 03:43:44 wessels Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -157,21 +157,6 @@ typedef enum { HDR_MISC_END } http_hdr_misc_t; -typedef struct proxy_ctrl_t { - int sock; - request_t *orig_request; - StoreEntry *entry; - peer *e; -} proxy_ctrl_t; - -typedef struct http_ctrl_t { - int sock; - request_t *request; - char *req_hdr; - int req_hdr_sz; - StoreEntry *entry; -} http_ctrl_t; - static char *HttpServerCCStr[] = { "public", @@ -213,20 +198,17 @@ static struct { int cc[SCC_ENUM_END]; } ReplyHeaderStats; -static void httpStateFree _PARAMS((int fd, void *)); -static PF httpTimeout; -static void httpMakePublic _PARAMS((StoreEntry *)); -static void httpMakePrivate _PARAMS((StoreEntry *)); -static void httpCacheNegatively _PARAMS((StoreEntry *)); -static void httpReadReply _PARAMS((int fd, void *)); -static void httpSendComplete _PARAMS((int fd, char *, int, int, void *)); -static void proxyhttpStartComplete _PARAMS((void *, int)); -static void httpStartComplete _PARAMS((void *, int)); -static void httpSendRequest _PARAMS((int fd, void *)); +static CNCB httpConnectDone; +static CWCB httpSendComplete; static IPH httpConnect; -static void httpConnectDone _PARAMS((int fd, int status, void *data)); +static PF httpReadReply; +static PF httpSendRequest; +static PF httpStateFree; +static PF httpTimeout; static void httpAppendRequestHeader _PARAMS((char *hdr, const char *line, size_t * sz, size_t max)); - +static void httpCacheNegatively _PARAMS((StoreEntry *)); +static void httpMakePrivate _PARAMS((StoreEntry *)); +static void httpMakePublic _PARAMS((StoreEntry *)); static void httpStateFree(int fd, void *data) @@ -857,8 +839,9 @@ proxyhttpStart(request_t * orig_request, StoreEntry * entry, peer * e) { - proxy_ctrl_t *ctrlp; - int sock; + HttpStateData *httpState; + request_t *request; + int fd; debug(11, 3, "proxyhttpStart: \"%s %s\"\n", RequestMethodStr[orig_request->method], entry->url); debug(11, 10, "proxyhttpStart: HTTP request header:\n%s\n", @@ -866,40 +849,18 @@ proxyhttpStart(request_t * orig_request, if (e->options & NEIGHBOR_PROXY_ONLY) storeStartDeleteBehind(entry); /* Create socket. */ - sock = comm_open(SOCK_STREAM, + fd = comm_open(SOCK_STREAM, 0, Config.Addrs.tcp_outgoing, 0, COMM_NONBLOCKING, entry->url); - if (sock == COMM_ERROR) { + if (fd == COMM_ERROR) { debug(11, 4, "proxyhttpStart: Failed because we're out of sockets.\n"); squid_error_entry(entry, ERR_NO_FDS, xstrerror()); return COMM_ERROR; } - ctrlp = xmalloc(sizeof(proxy_ctrl_t)); - ctrlp->sock = sock; - ctrlp->orig_request = orig_request; - ctrlp->entry = entry; - ctrlp->e = e; - storeLockObject(entry, proxyhttpStartComplete, ctrlp); - return COMM_OK; -} - - -static void -proxyhttpStartComplete(void *data, int status) -{ - proxy_ctrl_t *ctrlp = data; - int sock = ctrlp->sock; - request_t *orig_request = ctrlp->orig_request; - StoreEntry *entry = ctrlp->entry; - peer *e = ctrlp->e; - char *url = entry->url; - HttpStateData *httpState = NULL; - request_t *request = NULL; - int fd = ctrlp->sock; - xfree(ctrlp); + storeLockObject(entry, NULL, NULL); httpState = xcalloc(1, sizeof(HttpStateData)); httpState->entry = entry; httpState->req_hdr = entry->mem_obj->mime_hdr; @@ -908,7 +869,7 @@ proxyhttpStartComplete(void *data, int status) httpState->request = requestLink(request); httpState->neighbor = e; httpState->orig_request = requestLink(orig_request); - httpState->fd = sock; + httpState->fd = fd; /* register the handler to free HTTP state data when the FD closes */ comm_add_close_handler(httpState->fd, httpStateFree, @@ -917,14 +878,14 @@ proxyhttpStartComplete(void *data, int status) request->method = orig_request->method; xstrncpy(request->host, e->host, SQUIDHOSTNAMELEN); request->port = e->http_port; - xstrncpy(request->urlpath, url, MAX_URL); + xstrncpy(request->urlpath, entry->url, MAX_URL); BIT_SET(request->flags, REQ_PROXYING); httpState->ip_lookup_pending = 1; ipcache_nbgethostbyname(request->host, httpState->fd, httpConnect, httpState); - return; + return COMM_OK; } static void @@ -976,68 +937,40 @@ httpStart(request_t * request, int req_hdr_sz, StoreEntry * entry) { - http_ctrl_t *ctrlp; - int sock; + int fd; + HttpStateData *httpState; debug(11, 3, "httpStart: \"%s %s\"\n", RequestMethodStr[request->method], entry->url); debug(11, 10, "httpStart: req_hdr '%s'\n", req_hdr); /* Create socket. */ - sock = comm_open(SOCK_STREAM, + fd = comm_open(SOCK_STREAM, 0, Config.Addrs.tcp_outgoing, 0, COMM_NONBLOCKING, entry->url); - if (sock == COMM_ERROR) { + if (fd == COMM_ERROR) { debug(11, 4, "httpStart: Failed because we're out of sockets.\n"); squid_error_entry(entry, ERR_NO_FDS, xstrerror()); return COMM_ERROR; } - ctrlp = xmalloc(sizeof(http_ctrl_t)); - ctrlp->sock = sock; - ctrlp->request = request; - ctrlp->req_hdr = req_hdr; - ctrlp->req_hdr_sz = req_hdr_sz; - ctrlp->entry = entry; - storeLockObject(entry, httpStartComplete, ctrlp); - return COMM_OK; -} - - -static void -httpStartComplete(void *data, int status) -{ - http_ctrl_t *ctrlp = (http_ctrl_t *) data; - HttpStateData *httpState = NULL; - int sock; - request_t *request; - char *req_hdr; - int req_hdr_sz; - StoreEntry *entry; - - sock = ctrlp->sock; - request = ctrlp->request; - req_hdr = ctrlp->req_hdr; - req_hdr_sz = ctrlp->req_hdr_sz; - entry = ctrlp->entry; - xfree(ctrlp); - + storeLockObject(entry, NULL, NULL); httpState = xcalloc(1, sizeof(HttpStateData)); httpState->entry = entry; - httpState->req_hdr = req_hdr; httpState->req_hdr_sz = req_hdr_sz; httpState->request = requestLink(request); - httpState->fd = sock; + httpState->fd = fd; comm_add_close_handler(httpState->fd, httpStateFree, httpState); - commSetTimeout(sock, Config.Timeout.read, httpTimeout, httpState); + commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState); httpState->ip_lookup_pending = 1; ipcache_nbgethostbyname(request->host, httpState->fd, httpConnect, httpState); + return COMM_OK; } void diff --git a/src/ipcache.cc b/src/ipcache.cc index df28a733c4..3614f97f2b 100644 --- a/src/ipcache.cc +++ b/src/ipcache.cc @@ -1,6 +1,6 @@ /* - * $Id: ipcache.cc,v 1.113 1997/04/30 20:06:31 wessels Exp $ + * $Id: ipcache.cc,v 1.114 1997/05/05 03:43:46 wessels Exp $ * * DEBUG: section 14 IP Cache * AUTHOR: Harvest Derived @@ -146,7 +146,7 @@ static ipcache_entry *ipcacheAddNew _PARAMS((const char *, const struct hostent static void ipcacheAddHostent _PARAMS((ipcache_entry *, const struct hostent *)); static int ipcacheHasPending _PARAMS((ipcache_entry *)); static ipcache_entry *ipcache_get _PARAMS((const char *)); -static void dummy_handler _PARAMS((int, const ipcache_addrs *, void *)); +static IPH dummy_handler; static int ipcacheExpiredEntry _PARAMS((ipcache_entry *)); static void ipcacheAddPending _PARAMS((ipcache_entry *, int fd, IPH *, void *)); static void ipcacheEnqueue _PARAMS((ipcache_entry *)); @@ -162,6 +162,7 @@ static ipcache_addrs static_addrs; static HashID ip_table = 0; static struct ipcacheQueueData *ipcacheQueueHead = NULL; static struct ipcacheQueueData **ipcacheQueueTailP = &ipcacheQueueHead; +static int queue_length = 0; static char ipcache_status_char[] = { @@ -186,6 +187,7 @@ ipcacheEnqueue(ipcache_entry * i) new->i = i; *ipcacheQueueTailP = new; ipcacheQueueTailP = &new->next; + queue_length++; } static void * @@ -200,6 +202,7 @@ ipcacheDequeue(void) if (ipcacheQueueHead == NULL) ipcacheQueueTailP = &ipcacheQueueHead; safe_free(old); + queue_length--; } if (i && i->status != IP_PENDING) debug_trap("ipcacheDequeue: status != IP_PENDING"); @@ -546,7 +549,6 @@ ipcache_dnsHandleRead(int fd, void *data) { dnsserver_t *dnsData = data; int len; - int svc_time; int n; ipcache_entry *i = NULL; ipcache_entry *x = NULL; @@ -585,11 +587,9 @@ ipcache_dnsHandleRead(int fd, void *data) fatal_dump("ipcache_dnsHandleRead: bad status"); if (strstr(dnsData->ip_inbuf, "$end\n")) { /* end of record found */ - svc_time = tvSubMsec(dnsData->dispatch_time, current_time); - if (n > IPCACHE_AV_FACTOR) - n = IPCACHE_AV_FACTOR; - IpcacheStats.avg_svc_time - = (IpcacheStats.avg_svc_time * (n - 1) + svc_time) / n; + IpcacheStats.avg_svc_time = intAverage(IpcacheStats.avg_svc_time, + tvSubMsec(dnsData->dispatch_time, current_time), + n, IPCACHE_AV_FACTOR); if ((x = ipcache_parsebuffer(dnsData->ip_inbuf, dnsData)) == NULL) { debug(14, 0, "ipcache_dnsHandleRead: ipcache_parsebuffer failed?!\n"); } else { @@ -607,11 +607,6 @@ ipcache_dnsHandleRead(int fd, void *data) dnsData->data = NULL; dnsData->flags &= ~DNS_FLAG_BUSY; } - /* reschedule */ - commSetSelect(dnsData->inpipe, - COMM_SELECT_READ, - ipcache_dnsHandleRead, - dnsData, 0); ipcacheNudgeQueue(); } @@ -915,6 +910,7 @@ stat_ipcache_get(StoreEntry * sentry) IpcacheStats.release_locked); storeAppendPrintf(sentry, "{dnsserver avg service time: %d msec}\n", IpcacheStats.avg_svc_time); + storeAppendPrintf(sentry, "{pending queue length: %d}\n", queue_length); storeAppendPrintf(sentry, "}\n\n"); storeAppendPrintf(sentry, "{IP Cache Contents:\n\n"); storeAppendPrintf(sentry, " {%-29.29s %5s %6s %6s %1s}\n", diff --git a/src/neighbors.cc b/src/neighbors.cc index 94e277a3c3..b076cff19d 100644 --- a/src/neighbors.cc +++ b/src/neighbors.cc @@ -1,5 +1,5 @@ /* - * $Id: neighbors.cc,v 1.138 1997/05/01 04:25:41 wessels Exp $ + * $Id: neighbors.cc,v 1.139 1997/05/05 03:43:47 wessels Exp $ * * DEBUG: section 15 Neighbor Routines * AUTHOR: Harvest Derived @@ -586,11 +586,9 @@ neighborAlive(peer * p, const MemObject * mem, const icp_common_t * header) n = ++p->stats.pings_acked; if ((icp_opcode) header->opcode <= ICP_OP_END) p->stats.counts[header->opcode]++; - if (n > RTT_AV_FACTOR) - n = RTT_AV_FACTOR; if (mem) { rtt = tvSubMsec(mem->start_ping, current_time); - p->stats.rtt = (p->stats.rtt * (n - 1) + rtt) / n; + p->stats.rtt = intAverage(p->stats.rtt, rtt, n, RTT_AV_FACTOR); p->icp_version = (int) header->version; } } @@ -1090,16 +1088,11 @@ peerCountMcastPeersDone(void *data) ps_state *psstate = data; peer *p = psstate->callback_data; StoreEntry *fake = psstate->entry; - double old; - double new; - double D; p->mcast.flags &= ~PEER_COUNTING; - D = (double) ++p->mcast.n_times_counted; - if (D > 10.0) - D = 10.0; - old = p->mcast.avg_n_members; - new = (double) psstate->icp.n_recv; - p->mcast.avg_n_members = (old * (D - 1.0) + new) / D; + p->mcast.avg_n_members = doubleAverage(p->mcast.avg_n_members, + (double) psstate->icp.n_recv, + ++p->mcast.n_times_counted, + 10); debug(15, 1, "Group %s: %d replies, %4.1f average\n", p->host, psstate->icp.n_recv, diff --git a/src/redirect.cc b/src/redirect.cc index 30ce6067f8..ba6d41166c 100644 --- a/src/redirect.cc +++ b/src/redirect.cc @@ -1,5 +1,5 @@ /* - * $Id: redirect.cc,v 1.40 1997/04/30 22:46:26 wessels Exp $ + * $Id: redirect.cc,v 1.41 1997/05/05 03:43:48 wessels Exp $ * * DEBUG: section 29 Redirector * AUTHOR: Duane Wessels @@ -169,7 +169,6 @@ redirectHandleRead(int fd, void *data) redirectStateData *r = redirector->redirectState; char *t = NULL; int n; - int svc_time; len = read(fd, redirector->inbuf + redirector->offset, @@ -226,11 +225,10 @@ redirectHandleRead(int fd, void *data) redirector->flags &= ~REDIRECT_FLAG_BUSY; redirector->offset = 0; n = ++RedirectStats.replies; - svc_time = tvSubMsec(redirector->dispatch_time, current_time); - if (n > REDIRECT_AV_FACTOR) - n = REDIRECT_AV_FACTOR; - RedirectStats.avg_svc_time - = (RedirectStats.avg_svc_time * (n - 1) + svc_time) / n; + RedirectStats.avg_svc_time = + intAverage(RedirectStats.avg_svc_time, + tvSubMsec(redirector->dispatch_time, current_time), + n, REDIRECT_AV_FACTOR); } } while ((redirector = GetFirstAvailable()) && (r = Dequeue())) diff --git a/src/store.cc b/src/store.cc index 8c7f0feba8..37eaad0a0a 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.228 1997/05/02 21:34:15 wessels Exp $ + * $Id: store.cc,v 1.229 1997/05/05 03:43:49 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -596,13 +596,10 @@ storeReleaseRequest(StoreEntry * e) { if (BIT_TEST(e->flag, RELEASE_REQUEST)) return; - if (!storeEntryLocked(e)) { - debug_trap("Someone called storeReleaseRequest on an unlocked entry"); - debug(20, 0, " --> '%s'\n", e->url ? e->url : "NULL URL"); - return; - } - debug(20, 3, "storeReleaseRequest: FOR '%s'\n", e->key ? e->key : e->url); - e->flag |= RELEASE_REQUEST; + if (!storeEntryLocked(e)) + fatal_dump("storeReleaseRequest: unlocked entry"); + debug(20, 3, "storeReleaseRequest: '%s'\n", e->key); + BIT_SET(e->flag, RELEASE_REQUEST); storeSetPrivateKey(e); } @@ -618,9 +615,13 @@ storeUnlockObject(StoreEntry * e) if (e->lock_count) return (int) e->lock_count; if (e->store_status == STORE_PENDING) { - debug_trap("storeUnlockObject: Someone unlocked STORE_PENDING object"); - debug(20, 1, " --> Key '%s'\n", e->key); - e->store_status = STORE_ABORTED; + if (BIT_TEST(e->flag, ENTRY_DISPATCHED)) { + debug_trap("storeUnlockObject: PENDING and DISPATCHED with 0 locks"); + debug(20, 1, " --> Key '%s'\n", e->key); + e->store_status = STORE_ABORTED; + } else { + BIT_SET(e->flag, RELEASE_REQUEST); + } } if (storePendingNClients(e) > 0) debug_trap("storeUnlockObject: unlocked entry with pending clients\n"); @@ -853,7 +854,7 @@ storeAddDiskRestore(const char *url, int file_number, int size, time_t expires, { StoreEntry *e = NULL; - debug(20, 5, "StoreAddDiskRestore: '%s': size %d: expires %d: file_number %d\n", + debug(20, 5, "StoreAddDiskRestore: '%s': size %d: expires %d: fileno=%08X\n", url, size, expires, file_number); /* if you call this you'd better be sure file_number is not @@ -872,7 +873,6 @@ storeAddDiskRestore(const char *url, int file_number, int size, time_t expires, storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAP_OK; e->swap_file_number = file_number; - storeDirMapBitSet(file_number); e->object_len = size; e->lock_count = 0; BIT_RESET(e->flag, CLIENT_ABORT_REQUEST); @@ -882,10 +882,14 @@ storeAddDiskRestore(const char *url, int file_number, int size, time_t expires, e->expires = expires; e->lastmod = lastmod; e->ping_status = PING_NONE; - if (store_rebuilding == STORE_REBUILDING_CLEAN) + if (store_rebuilding == STORE_REBUILDING_CLEAN) { BIT_SET(e->flag, ENTRY_VALIDATED); - else + /* Only set the file bit if we know its a valid entry */ + /* otherwise, set it in the validation procedure */ + storeDirMapBitSet(file_number); + } else { BIT_RESET(e->flag, ENTRY_VALIDATED); + } return e; } @@ -1098,6 +1102,8 @@ storeSwapInHandle(int u1, const char *buf, int len, int flag, void *data) e); return; } + if (mem->e_current_len > e->object_len) + debug_trap("storeSwapInHandle: Too much data read!"); /* complete swapping in */ storeSetMemStatus(e, IN_MEMORY); put_free_8k_page(mem->e_swap_buf); @@ -1106,7 +1112,7 @@ storeSwapInHandle(int u1, const char *buf, int len, int flag, void *data) debug(20, 5, "storeSwapInHandle: SwapIn complete: '%s' from %s.\n", e->url, storeSwapFullPath(e->swap_file_number, NULL)); if (mem->e_current_len != e->object_len) { - debug(20, 0, "storeSwapInHandle: WARNING: Object size mismatch.\n"); + debug_trap("storeSwapInHandle: Object size mismatch"); debug(20, 0, " --> '%s'\n", e->url); debug(20, 0, " --> Expecting %d bytes from file: %s\n", e->object_len, storeSwapFullPath(e->swap_file_number, NULL)); @@ -1230,7 +1236,6 @@ storeSwapOutHandle(int fd, int flag, void *data) put_free_8k_page(mem->e_swap_buf); file_close(fd); if (e->swap_file_number != -1) { - storeDirMapBitReset(e->swap_file_number); storePutUnusedFileno(e->swap_file_number); e->swap_file_number = -1; } @@ -1443,10 +1448,8 @@ storeDoRebuildFromDisk(void *data) if (x < 1) continue; storeSwapFullPath(sfileno, swapfile); - if (x != 6) { - storePutUnusedFileno(sfileno); + if (x != 6) continue; - } if (sfileno < 0) continue; sfileno = storeDirProperFileno(d->dirn, sfileno); @@ -1455,18 +1458,6 @@ storeDoRebuildFromDisk(void *data) lastmod = (time_t) scan3; size = (off_t) scan4; - if ((e = storeGet(url))) { - if (e->timestamp > timestamp) { - /* already have a newer object in memory, throw old one away */ - debug(20, 3, "storeRebuildFromDisk: Replaced: %s\n", url); - RB->dupcount++; - continue; - } - debug(20, 6, "storeRebuildFromDisk: Duplicate: '%s'\n", url); - storeRelease(e); - RB->objcount--; - RB->dupcount++; - } e = storeGet(url); used = storeDirMapBitTest(sfileno); /* If this URL already exists in the cache, does the swap log @@ -1630,10 +1621,12 @@ storeValidateComplete(void *data, int retcode, int errcode) path = storeSwapFullPath(e->swap_file_number, NULL); retcode = stat(path, sb); } - if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) + if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { BIT_RESET(e->flag, ENTRY_VALIDATED); - else + } else { BIT_SET(e->flag, ENTRY_VALIDATED); + storeDirMapBitSet(e->swap_file_number); + } errno = errcode; (ctrlp->callback) (ctrlp->callback_data, retcode); xfree(sb); @@ -2191,9 +2184,9 @@ storeRelease(StoreEntry * e) debug(20, 5, "storeRelease: Release anonymous object\n"); if (e->swap_status == SWAP_OK && (e->swap_file_number > -1)) { - storePutUnusedFileno(e->swap_file_number); + if (BIT_TEST(e->flag, ENTRY_VALIDATED)) + storePutUnusedFileno(e->swap_file_number); storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); - storeDirMapBitReset(e->swap_file_number); e->swap_file_number = -1; HTTPCacheInfo->proto_purgeobject(HTTPCacheInfo, urlParseProtocol(e->url), @@ -2601,9 +2594,9 @@ storeWriteCleanLogs(void) safeunlink(new[dirn], 1); safeunlink(cln[dirn], 1); fd[dirn] = file_open(new[dirn], - O_WRONLY | O_CREAT | O_TRUNC, - NULL, - NULL); + O_WRONLY | O_CREAT | O_TRUNC, + NULL, + NULL); if (fd[dirn] < 0) { debug(50, 0, "storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); continue; @@ -2873,6 +2866,7 @@ storeFreeMemory(void) xfree(list); hashFreeMemory(store_table); safe_free(MaintBucketsOrder); + storeDirCloseSwapLogs(); } int @@ -2932,14 +2926,20 @@ int fileno_stack_count = 0; static int storeGetUnusedFileno(void) { + int fn; if (fileno_stack_count < 1) return -1; - return fileno_stack[--fileno_stack_count]; + fn = fileno_stack[--fileno_stack_count]; + storeDirMapBitSet(fn); + return fn; } static void storePutUnusedFileno(int fileno) { + if (!storeDirMapBitTest(fileno)) + fatal_dump("storePutUnusedFileno: fileno not in use"); + storeDirMapBitReset(fileno); if (fileno_stack_count < FILENO_STACK_SIZE) fileno_stack[fileno_stack_count++] = fileno; else diff --git a/src/store_dir.cc b/src/store_dir.cc index bc2e24bbcb..a6a0248574 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -190,10 +190,12 @@ storeDirMapBitReset(int fn) int storeDirMapAllocate(void) { + int fn; int dirn = storeMostFreeSwapDir(); SwapDir *SD = &SwapDirs[dirn]; int filn = file_map_allocate(SD->map, SD->suggest); - return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK); + fn = (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK); + return fn; } char * @@ -314,7 +316,7 @@ storeDirOpenTmpSwapLog(int dirn, int *clean_flag) if (SD->swaplog_fd >= 0) file_close(SD->swaplog_fd); /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT, NULL, NULL); + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL); if (fd < 0) { debug(50, 1, "%s: %s\n", new_path, xstrerror()); fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); diff --git a/src/tools.cc b/src/tools.cc index 5cdff1e893..bfb4c2a1b3 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -1,6 +1,6 @@ /* - * $Id: tools.cc,v 1.103 1997/05/02 21:34:17 wessels Exp $ + * $Id: tools.cc,v 1.104 1997/05/05 03:43:50 wessels Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -174,7 +174,7 @@ mail_warranty(void) } static void -dumpMallocStats(FILE * f) +dumpMallocStats() { #if HAVE_MALLINFO struct mallinfo mp; @@ -182,37 +182,37 @@ dumpMallocStats(FILE * f) if (!do_mallinfo) return; mp = mallinfo(); - fprintf(f, "Memory usage for %s via mallinfo():\n", appname); - fprintf(f, "\ttotal space in arena: %6d KB\n", + fprintf(debug_log, "Memory usage for %s via mallinfo():\n", appname); + fprintf(debug_log, "\ttotal space in arena: %6d KB\n", mp.arena >> 10); - fprintf(f, "\tOrdinary blocks: %6d KB %6d blks\n", + fprintf(debug_log, "\tOrdinary blocks: %6d KB %6d blks\n", mp.uordblks >> 10, mp.ordblks); - fprintf(f, "\tSmall blocks: %6d KB %6d blks\n", + fprintf(debug_log, "\tSmall blocks: %6d KB %6d blks\n", mp.usmblks >> 10, mp.smblks); - fprintf(f, "\tHolding blocks: %6d KB %6d blks\n", + fprintf(debug_log, "\tHolding blocks: %6d KB %6d blks\n", mp.hblkhd >> 10, mp.hblks); - fprintf(f, "\tFree Small blocks: %6d KB\n", + fprintf(debug_log, "\tFree Small blocks: %6d KB\n", mp.fsmblks >> 10); - fprintf(f, "\tFree Ordinary blocks: %6d KB\n", + fprintf(debug_log, "\tFree Ordinary blocks: %6d KB\n", mp.fordblks >> 10); t = mp.uordblks + mp.usmblks + mp.hblkhd; - fprintf(f, "\tTotal in use: %6d KB %d%%\n", + fprintf(debug_log, "\tTotal in use: %6d KB %d%%\n", t >> 10, percent(t, mp.arena)); t = mp.fsmblks + mp.fordblks; - fprintf(f, "\tTotal free: %6d KB %d%%\n", + fprintf(debug_log, "\tTotal free: %6d KB %d%%\n", t >> 10, percent(t, mp.arena)); #if HAVE_EXT_MALLINFO - fprintf(f, "\tmax size of small blocks:\t%d\n", + fprintf(debug_log, "\tmax size of small blocks:\t%d\n", mp.mxfast); - fprintf(f, "\tnumber of small blocks in a holding block:\t%d\n", + fprintf(debug_log, "\tnumber of small blocks in a holding block:\t%d\n", mp.nlblks); - fprintf(f, "\tsmall block rounding factor:\t%d\n", + fprintf(debug_log, "\tsmall block rounding factor:\t%d\n", mp.grain); - fprintf(f, "\tspace (including overhead) allocated in ord. blks:\t%d\n", + fprintf(debug_log, "\tspace (including overhead) allocated in ord. blks:\t%d\n", mp.uordbytes); - fprintf(f, "\tnumber of ordinary blocks allocated:\t%d\n", + fprintf(debug_log, "\tnumber of ordinary blocks allocated:\t%d\n", mp.allocated); - fprintf(f, "\tbytes used in maintaining the free tree:\t%d\n", + fprintf(debug_log, "\tbytes used in maintaining the free tree:\t%d\n", mp.treeoverhead); #endif /* HAVE_EXT_MALLINFO */ #if PRINT_MMAP @@ -221,8 +221,8 @@ dumpMallocStats(FILE * f) #endif /* HAVE_MALLINFO */ } -static int -PrintRusage(void (*f) (void), FILE * lf) +static void +PrintRusage(void) { #if HAVE_GETRUSAGE && defined(RUSAGE_SELF) struct rusage rusage; @@ -234,21 +234,17 @@ PrintRusage(void (*f) (void), FILE * lf) #ifdef _SQUID_SOLARIS_ leave_suid(); #endif - fprintf(lf, "CPU Usage: user %d sys %d\n", + fprintf(debug_log, "CPU Usage: user %d sys %d\n", (int) rusage.ru_utime.tv_sec, (int) rusage.ru_stime.tv_sec); #if defined(_SQUID_SGI_) || defined(_SQUID_OSF_) || defined(BSD4_4) - fprintf(lf, "Maximum Resident Size: %ld KB\n", rusage.ru_maxrss); + fprintf(debug_log, "Maximum Resident Size: %ld KB\n", rusage.ru_maxrss); #else /* _SQUID_SGI_ */ - fprintf(lf, "Maximum Resident Size: %ld KB\n", + fprintf(debug_log, "Maximum Resident Size: %ld KB\n", (rusage.ru_maxrss * getpagesize()) >> 10); #endif /* _SQUID_SGI_ */ - fprintf(lf, "Page faults with physical i/o: %ld\n", + fprintf(debug_log, "Page faults with physical i/o: %ld\n", rusage.ru_majflt); #endif /* HAVE_GETRUSAGE */ - dumpMallocStats(lf); - if (f) - f(); - return 0; } void @@ -288,7 +284,8 @@ death(int sig) #endif releaseServerSockets(); storeWriteCleanLogs(); - PrintRusage(NULL, debug_log); + PrintRusage(); + dumpMallocStats(); if (squid_curtime - SQUID_RELEASE_TIME < 864000) { /* skip if more than 10 days old */ if (Config.adminEmail) @@ -356,13 +353,13 @@ normal_shutdown(void) releaseServerSockets(); unlinkdClose(); storeWriteCleanLogs(); - PrintRusage(NULL, debug_log); + PrintRusage(); + dumpMallocStats(); storeCloseLog(); statCloseLog(); #if PURIFY configFreeMemory(); storeFreeMemory(); - fdFreeMemory(); dnsFreeMemory(); redirectFreeMemory(); errorpageFreeMemory(); @@ -371,6 +368,11 @@ normal_shutdown(void) ipcacheFreeMemory(); fqdncacheFreeMemory(); #endif + file_close(0); + file_close(1); + file_close(2); + fdDumpOpen(); + fdFreeMemory(); debug(21, 0, "Squid Cache (Version %s): Exiting normally.\n", version_string); fclose(debug_log); @@ -388,7 +390,8 @@ fatal_common(const char *message) fprintf(debug_log, "Squid Cache (Version %s): Terminated abnormally.\n", version_string); fflush(debug_log); - PrintRusage(NULL, debug_log); + PrintRusage(); + dumpMallocStats(); } /* fatal */ @@ -582,24 +585,27 @@ no_suid(void) void writePidFile(void) { - FILE *pid_fp = NULL; + int fd; const char *f = NULL; mode_t old_umask; - - if ((f = Config.pidFilename) == NULL || !strcmp(Config.pidFilename, "none")) + char buf[32]; + if ((f = Config.pidFilename) == NULL) + return; + if (!strcmp(Config.pidFilename, "none")) return; enter_suid(); old_umask = umask(022); - pid_fp = fopen(f, "w"); + fd = file_open(f, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL); umask(old_umask); leave_suid(); - if (pid_fp != NULL) { - fprintf(pid_fp, "%d\n", (int) getpid()); - fclose(pid_fp); - } else { - debug(50, 0, "WARNING: Could not write pid file\n"); - debug(50, 0, " %s: %s\n", f, xstrerror()); + if (fd < 0) { + debug(50, 0, "%s: %s\n", f, xstrerror()); + debug_trap("Could not write pid file"); + return; } + sprintf(buf, "%d\n", (int) getpid()); + write(fd, buf, strlen(buf)); + file_close(fd); } @@ -607,11 +613,11 @@ pid_t readPidFile(void) { FILE *pid_fp = NULL; - const char *f = NULL; + const char *f = Config.pidFilename; pid_t pid = -1; int i; - if ((f = Config.pidFilename) == NULL || !strcmp(Config.pidFilename, "none")) { + if (f == NULL || !strcmp(Config.pidFilename, "none")) { fprintf(stderr, "%s: ERROR: No pid file name defined\n", appname); exit(1); } @@ -731,3 +737,18 @@ inaddrFromHostent(const struct hostent *hp) xmemcpy(&s.s_addr, hp->h_addr, sizeof(s.s_addr)); return s; } +double +doubleAverage(double cur, double new, int n, int max) +{ + if (n > max) + n = max; + return (cur * (n - 1)) + new / n; +} + +int +intAverage(int cur, int new, int n, int max) +{ + if (n > max) + n = max; + return (cur * (n - 1)) + new / n; +}