From: wessels <> Date: Tue, 3 Feb 1998 04:14:49 +0000 (+0000) Subject: Stewart Forster Jumbo Patch X-Git-Tag: SQUID_3_0_PRE1~4202 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6cf028abeec045f1f05709a060b2c1b42cc22b88;p=thirdparty%2Fsquid.git Stewart Forster Jumbo Patch - Completely rewritten aiops.c that creates and manages a pool of threads so thread creation overhead is eliminated - Lots of extra debugging - Lots of mods to store.c to detect and cancel outstanding ASYNC ops. Code is not proven exhaustive and there are definately still cases to be found where outstanding disk ops aren't cancelled properly. - Changes to call interface to a few routines to support disk op `tagging', so operations can be cleanly cancelled on store_abort()s. - Implementation of swap.state files as transaction logs. Removed objects are now noted with a negative object size. This allows reliatively clean rebuilds from non-clean shutdowns. - Now that the swap.state files are transaction logs, there's now no need to validate by stat()ing. All the validation procedure does is now just set the valid bit AFTER all the swap.state files have been read, because by that time, only valid objects can be left. Object still need to be marked invalid when reading the swap.state file because there's no guarantee the file has been retaken or deleted. - An fstat() call is now added after every storeSwapInFileOpened() so object sizes can be checked. Added code to storeRelease() the object if the sizes don't match. - #defining USE_ASYNC_IO now uses the async unlink() rather than unlinkd() - #defining MONOTONIC_STORE will support the creation of disk objects clustered into directories. This GREATLY improves disk performance (factor of 3) over old `write-over-old-object' method. If using the MONOTONIC_STORE, the {get/put}_unusedFileno stack stuff is disabled. This is actually a good thing and greatly reduces the risk of serving up bad objects. - Fixed unlink() in storeWriteCleanLogs to be real unlink() rather than ASYNC/unlinkd unlinks. swap.state.new files were being removed just after they were created due to delayed unlinks. - Disabled various assertions and made these into debug warning messages to make the code more stable until the bugs can be tracked down. - Added most of Michael O'Reilly's patches which included many bug fixes. Ask him for full details. - Moved aio_check_callbacks in comm_{poll|select}(). It was called after the fdset had been built which was wrong because the callbacks were changing the state of the read/write handlers prior to the poll/select() calls. --- diff --git a/ChangeLog b/ChangeLog index 51393230b1..f7941ebaf8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,7 +23,51 @@ - Use memmove/bcopy as detected by configure Some systems does not have memmove, but have the older bcopy implementation (Henrik Nordstrom). - + - Completely rewritten aiops.c that creates and manages a pool + of threads so thread creation overhead is eliminated (SLF). + - Lots of mods to store.c to detect and cancel outstanding + ASYNC ops. Code is not proven exhaustive and there are + definately still cases to be found where outstanding disk ops + aren't cancelled properly (SLF). + - Changes to call interface to a few routines to support disk + op `tagging', so operations can be cleanly cancelled on + store_abort()s (SLF). + - Implementation of swap.state files as transaction logs. + Removed objects are now noted with a negative object size. + This allows reliatively clean rebuilds from non-clean + shutdowns (SLF). + - Now that the swap.state files are transaction logs, there's + now no need to validate by stat()ing. All the validation + procedure does is now just set the valid bit AFTER all the + swap.state files have been read, because by that time, only + valid objects can be left. Object still need to be marked + invalid when reading the swap.state file because there's no + guarantee the file has been retaken or deleted (SLF). + - An fstat() call is now added after every + storeSwapInFileOpened() so object sizes can be checked. Added + code to storeRelease() the object if the sizes don't match (SLF). + - #defining USE_ASYNC_IO now uses the async unlink() rather than + unlinkd() (SLF). + - #defining MONOTONIC_STORE will support the creation of disk + objects clustered into directories. This GREATLY improves disk + performance (factor of 3) over old `write-over-old-object' + method. If using the MONOTONIC_STORE, the + {get/put}_unusedFileno stack stuff is disabled. This is + actually a good thing and greatly reduces the risk of serving + up bad objects (SLF). + - Fixed unlink() in storeWriteCleanLogs to be real unlink() + rather than ASYNC/unlinkd unlinks. swap.state.new files were + being removed just after they were created due to delayed + unlinks (SLF). + - Disabled various assertions and made these into debug warning + messages to make the code more stable until the bugs can be + tracked down (SLF). + - Added most of Michael O'Reilly's patches which included many + bug fixes. Ask him for full details (SLF). + - Moved aio_check_callbacks in comm_{poll|select}(). It was + called after the fdset had been built which was wrong because + the callbacks were changing the state of the read/write + handlers prior to the poll/select() calls (SLF). Changes to squid-1.2.beta12 (Jan 30, 1998): diff --git a/src/access_log.cc b/src/access_log.cc index e1a5d1a115..d38f14f4cc 100644 --- a/src/access_log.cc +++ b/src/access_log.cc @@ -1,7 +1,7 @@ /* - * $Id: access_log.cc,v 1.18 1998/02/02 07:20:53 wessels Exp $ + * $Id: access_log.cc,v 1.19 1998/02/02 21:14:54 wessels Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -190,7 +190,7 @@ accessLogOpen(const char *fname) { assert(fname); xstrncpy(LogfileName, fname, SQUID_MAXPATHLEN); - LogfileFD = file_open(LogfileName, O_WRONLY | O_CREAT, NULL, NULL); + LogfileFD = file_open(LogfileName, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (LogfileFD == DISK_ERROR) { debug(50, 0) ("%s: %s\n", LogfileName, xstrerror()); fatal("Cannot open logfile."); @@ -287,7 +287,7 @@ accessLogRotate(void) /* Close and reopen the log. It may have been renamed "manually" * before HUP'ing us. */ file_close(LogfileFD); - LogfileFD = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); + LogfileFD = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (LogfileFD == DISK_ERROR) { debug(46, 0) ("accessLogRotate: Cannot open logfile: %s\n", fname); LogfileStatus = LOG_DISABLE; diff --git a/src/asn.cc b/src/asn.cc index 83cc9ba234..27f7ab78d1 100644 --- a/src/asn.cc +++ b/src/asn.cc @@ -1,5 +1,5 @@ /* - * $Id: asn.cc,v 1.14 1998/01/12 04:30:35 wessels Exp $ + * $Id: asn.cc,v 1.15 1998/02/02 21:14:55 wessels Exp $ * * DEBUG: section 53 AS Number handling * AUTHOR: Duane Wessels, Kostas Anagnostakis @@ -215,6 +215,7 @@ asHandleReply(void *data, char *buf, ssize_t size) memFree(MEM_4K_BUF, buf); return; } else if (size < 0) { + debug(50, 1) ("asHandleReply: Called with size=%d.\n", size); memFree(MEM_4K_BUF, buf); return; } diff --git a/src/client_side.cc b/src/client_side.cc index b8e4fa49e7..c0f929b7d9 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.200 1998/02/02 19:50:09 wessels Exp $ + * $Id: client_side.cc,v 1.201 1998/02/02 21:14:57 wessels Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -245,6 +245,11 @@ clientRedirectDone(void *data, char *result) new_request->http_ver = old_request->http_ver; new_request->headers = xstrdup(old_request->headers); new_request->headers_sz = old_request->headers_sz; + if (old_request->body) { + new_request->body = xmalloc(old_request->body_sz); + xmemcpy(new_request->body, old_request->body, old_request->body_sz); + new_request->body_sz = old_request->body_sz; + } requestUnlink(old_request); http->request = requestLink(new_request); urlCanonical(http->request, http->uri); @@ -278,6 +283,8 @@ clientProcessExpired(void *data) http->out.offset = 0; protoDispatch(http->conn->fd, http->entry, http->request); /* Register with storage manager to receive updates when data comes in. */ + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientProcessExpired: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset, http->out.offset, @@ -337,6 +344,8 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) entry->refcount++; } else if (mem->reply->code == 0) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientHandleIMSReply: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset + size, http->out.offset, @@ -384,6 +393,12 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) /* use clientCacheHit() here as the callback because we might * be swapping in from disk, and the file might not really be * there */ + if(entry->store_status == STORE_ABORTED) { + debug(33, 0) ("clientHandleIMSReply: IMS swapin failed on aborted object\n"); + http->log_type = LOG_TCP_SWAPFAIL_MISS; + clientProcessMiss(http); + return; + } storeClientCopy(entry, http->out.offset, http->out.offset, @@ -855,7 +870,10 @@ clientCacheHit(void *data, char *buf, ssize_t size) debug(12, 3) ("clientCacheHit: request aborted\n"); } else { /* swap in failure */ + debug(12, 1) ("KARMA: clientCacheHit fail\n"); http->log_type = LOG_TCP_SWAPFAIL_MISS; + if (http->entry) + storeRelease(http->entry); clientProcessMiss(http); } } @@ -927,6 +945,10 @@ clientSendMoreData(void *data, char *buf, ssize_t size) */ debug(12, 3) ("clientSendMoreData: Appending %d bytes after headers\n", (int) (size - hdrlen)); + if (((size - hdrlen) + l) > 8192) { + debug(0,0) ("Size, hdrlen, l %d, %d, %d\n", size, hdrlen, l); + return; + } xmemcpy(newbuf + l, buf + hdrlen, size - hdrlen); /* replace buf with newbuf */ freefunc(buf); @@ -1000,7 +1022,9 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da httpRequestFree(http); if ((http = conn->chr) != NULL) { debug(12, 1) ("clientWriteComplete: FD %d Sending next request\n", fd); - if (!storeClientCopyPending(http->entry, http)) + if (!storeClientCopyPending(http->entry, http)) { + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientWriteComplete: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(http->entry, http->out.offset, http->out.offset, @@ -1008,6 +1032,7 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da memAllocate(MEM_4K_BUF, 1), clientSendMoreData, http); + } } else { debug(12, 5) ("clientWriteComplete: FD %d reading next request\n", fd); fd_note(fd, "Reading next request"); @@ -1026,6 +1051,8 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da } else { /* More data will be coming from primary server; register with * storage manager. */ + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientWriteComplete 2: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset, http->out.offset, @@ -1046,7 +1073,7 @@ clientGetHeadersForIMS(void *data, char *buf, ssize_t size) assert(size <= SM_PAGE_SIZE); memFree(MEM_4K_BUF, buf); buf = NULL; - if (size < 0) { + if (size < 0 || entry->store_status == STORE_ABORTED) { debug(12, 1) ("clientGetHeadersForIMS: storeClientCopy failed for '%s'\n", storeKeyText(entry->key)); clientProcessMiss(http); @@ -1058,6 +1085,8 @@ clientGetHeadersForIMS(void *data, char *buf, ssize_t size) return; } /* All headers are not yet available, wait for more data */ + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientGetHeadersForIMS: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset + size, http->out.offset, @@ -1092,6 +1121,8 @@ clientGetHeadersForIMS(void *data, char *buf, ssize_t size) http->log_type = LOG_TCP_IMS_HIT; entry->refcount++; if (modifiedSince(entry, http->request)) { + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientGetHeadersForIMS 2: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset, http->out.offset, @@ -1242,6 +1273,8 @@ clientProcessRequest(clientHttpRequest * http) case LOG_TCP_NEGATIVE_HIT: case LOG_TCP_MEM_HIT: entry->refcount++; /* HIT CASE */ + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientProcessRequest: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset, http->out.offset, @@ -1251,6 +1284,8 @@ clientProcessRequest(clientHttpRequest * http) http); break; case LOG_TCP_IMS_MISS: + if(entry->store_status == STORE_ABORTED) + debug(33, 0) ("clientProcessRequest 2: entry->swap_status == STORE_ABORTED\n"); storeClientCopy(entry, http->out.offset, http->out.offset, diff --git a/src/comm.cc b/src/comm.cc index a06710f33d..31c700489d 100644 --- a/src/comm.cc +++ b/src/comm.cc @@ -1,6 +1,6 @@ /* - * $Id: comm.cc,v 1.222 1998/01/12 04:29:58 wessels Exp $ + * $Id: comm.cc,v 1.223 1998/02/02 21:14:58 wessels Exp $ * * DEBUG: section 5 Socket Functions * AUTHOR: Harvest Derived @@ -328,6 +328,10 @@ commConnectDnsHandle(const ipcache_addrs * ia, void *data) cs->locks--; if (ia == NULL) { debug(5, 3) ("commConnectDnsHandle: Unknown host: %s\n", cs->host); + if (!dns_error_message) { + dns_error_message = "Unknown DNS error"; + debug(5,1) ("commConnectDnsHandle: Bad dns_error_message\n"); + } assert(dns_error_message != NULL); commConnectCallback(cs, COMM_ERR_DNS); return; @@ -621,6 +625,8 @@ comm_close(int fd) F = &fd_table[fd]; if (EBIT_TEST(F->flags, FD_CLOSING)) return; + if(shutdown_pending && (!F->open || F->type == FD_FILE)) + return; assert(F->open); assert(F->type != FD_FILE); EBIT_SET(F->flags, FD_CLOSING); @@ -733,14 +739,18 @@ comm_poll_incoming(void) if (((revents = pfds[i].revents) == 0) || ((fd = pfds[i].fd) == -1)) continue; if (revents & (POLLRDNORM | POLLIN | POLLHUP | POLLERR)) { - hdl = fd_table[fd].read_handler; - fd_table[fd].read_handler = NULL; - hdl(fd, fd_table[fd].read_data); + if(hdl = fd_table[fd].read_handler) { + fd_table[fd].read_handler = NULL; + hdl(fd, fd_table[fd].read_data); + } else + debug(5,1) ("comm_poll_incoming: NULL read handler\n"); } if (revents & (POLLWRNORM | POLLOUT | POLLHUP | POLLERR)) { - hdl = fd_table[fd].write_handler; - fd_table[fd].write_handler = NULL; - hdl(fd, fd_table[fd].write_data); + if(hdl = fd_table[fd].write_handler) { + fd_table[fd].write_handler = NULL; + hdl(fd, fd_table[fd].write_data); + } else + debug(5,1) ("comm_poll_incoming: NULL write handler\n"); } } /* TO FIX: repoll ICP connection here */ @@ -801,14 +811,20 @@ comm_select_incoming(void) for (i = 0; i < N; i++) { fd = fds[i]; if (FD_ISSET(fd, &read_mask)) { - hdl = fd_table[fd].read_handler; - fd_table[fd].read_handler = NULL; - hdl(fd, fd_table[fd].read_data); + if((hdl = fd_table[fd].read_handler) != NULL) { + fd_table[fd].read_handler = NULL; + hdl(fd, fd_table[fd].read_data); + } else { + debug(5,1) ("comm_select_incoming: NULL read handler\n"); + } } if (FD_ISSET(fd, &write_mask)) { - hdl = fd_table[fd].write_handler; - fd_table[fd].write_handler = NULL; - hdl(fd, fd_table[fd].write_data); + if((hdl = fd_table[fd].write_handler) != NULL) { + fd_table[fd].write_handler = NULL; + hdl(fd, fd_table[fd].write_data); + } else { + debug(5,1) ("comm_select_incoming: NULL write handler\n"); + } } } } @@ -864,6 +880,9 @@ comm_poll(time_t sec) else setSocketShutdownLifetimes(1); } +#if USE_ASYNC_IO + aioCheckCallbacks(); +#endif nfds = 0; maxfd = Biggest_FD + 1; for (i = 0; i < maxfd; i++) { @@ -887,7 +906,6 @@ comm_poll(time_t sec) return COMM_SHUTDOWN; #if USE_ASYNC_IO poll_time = sec > 0 ? 50 : 0; - aioCheckCallbacks(); #else poll_time = sec > 0 ? 1000 : 0; #endif @@ -961,6 +979,8 @@ comm_poll(time_t sec) F->timeout_handler = NULL; F->read_handler = NULL; F->write_handler = NULL; + if(F->open != 0) + fd_close(fd); } lastinc = polledinc; } @@ -999,6 +1019,11 @@ comm_select(time_t sec) #if !ALARM_UPDATES_TIME getCurrentTime(); #endif + +#if USE_ASYNC_IO + aioCheckCallbacks(); +#endif + FD_ZERO(&readfds); FD_ZERO(&writefds); if (shutdown_pending || reconfigure_pending) { @@ -1031,9 +1056,6 @@ comm_select(time_t sec) debug(5, 2) ("comm_select: Still waiting on %d FDs\n", nfds); if (nfds == 0) return COMM_SHUTDOWN; -#if USE_ASYNC_IO - aioCheckCallbacks(); -#endif for (;;) { poll_time.tv_sec = sec > 0 ? 1 : 0; poll_time.tv_usec = 0; @@ -1394,6 +1416,14 @@ comm_write(int fd, char *buf, int size, CWCB * handler, void *handler_data, FREE CommWriteStateData *state = NULL; debug(5, 5) ("comm_write: FD %d: sz %d: hndl %p: data %p.\n", fd, size, handler, handler_data); + if (fd_table[fd].rwstate) { + debug(5,1) ("comm_write: fd_table[%d].rwstate != NULL", fd); + state = fd_table[fd].rwstate; + debug(5,1) ("comm_write: %d'%s',(%d,%d)'%s'\n", size, buf, state->size, + state->offset, state->buf); + safe_free(fd_table[fd].rwstate); + fd_table[fd].rwstate = NULL; + } assert(fd_table[fd].rwstate == NULL); state = xcalloc(1, sizeof(CommWriteStateData)); state->buf = buf; diff --git a/src/disk.cc b/src/disk.cc index 4d91ddd045..b6d415ccbb 100644 --- a/src/disk.cc +++ b/src/disk.cc @@ -1,6 +1,6 @@ /* - * $Id: disk.cc,v 1.96 1998/02/02 07:20:54 wessels Exp $ + * $Id: disk.cc,v 1.97 1998/02/02 21:14:59 wessels Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -135,7 +135,7 @@ disk_init(void) /* Open a disk file. Return a file descriptor */ int -file_open(const char *path, int mode, FOCB * callback, void *callback_data) +file_open(const char *path, int mode, FOCB * callback, void *callback_data, void *tag) { int fd; open_ctrl_t *ctrlp; @@ -152,7 +152,7 @@ file_open(const char *path, int mode, FOCB * callback, void *callback_data) /* Open file */ #if USE_ASYNC_IO if (callback != NULL) { - aioOpen(path, mode, 0644, file_open_complete, ctrlp); + aioOpen(path, mode, 0644, file_open_complete, ctrlp, tag); return DISK_OK; } #endif @@ -168,12 +168,20 @@ static void file_open_complete(void *data, int fd, int errcode) { open_ctrl_t *ctrlp = (open_ctrl_t *) data; + + if(fd == -2 && errcode == -2) { /* Cancelled - clean up */ + if (ctrlp->callback) + (ctrlp->callback) (ctrlp->callback_data, fd, errcode); + xfree(ctrlp->path); + xfree(ctrlp); + return; + } if (fd < 0) { errno = errcode; debug(50, 0) ("file_open: error opening file %s: %s\n", ctrlp->path, xstrerror()); if (ctrlp->callback) - (ctrlp->callback) (ctrlp->callback_data, DISK_ERROR); + (ctrlp->callback) (ctrlp->callback_data, DISK_ERROR, errcode); xfree(ctrlp->path); xfree(ctrlp); return; @@ -182,7 +190,7 @@ file_open_complete(void *data, int fd, int errcode) commSetCloseOnExec(fd); fd_open(fd, FD_FILE, ctrlp->path); if (ctrlp->callback) - (ctrlp->callback) (ctrlp->callback_data, fd); + (ctrlp->callback) (ctrlp->callback_data, fd, errcode); xfree(ctrlp->path); xfree(ctrlp); } @@ -192,7 +200,10 @@ void file_close(int fd) { fde *F = &fd_table[fd]; - assert(fd >= 0); + if(fd < 0) { + debug(6, 0) ("file_close: FD less than zero: %d\n", fd); + return; + } assert(F->open); if (EBIT_TEST(F->flags, FD_WRITE_DAEMON)) { EBIT_SET(F->flags, FD_CLOSE_REQUEST); @@ -202,13 +213,13 @@ file_close(int fd) EBIT_SET(F->flags, FD_CLOSE_REQUEST); return; } - fd_close(fd); - debug(6, 5) ("file_close: FD %d\n", fd); #if USE_ASYNC_IO aioClose(fd); #else close(fd); #endif + debug(6, 5) ("file_close: FD %d\n", fd); + fd_close(fd); } @@ -277,16 +288,28 @@ diskHandleWriteComplete(void *data, int len, int errcode) dwrite_q *q = fdd->write_q; int status = DISK_OK; errno = errcode; + safe_free(data); - fd_bytes(fd, len, FD_WRITE); if (q == NULL) /* Someone aborted then write completed */ return; + + if(len == -2 && errcode == -2) { /* Write cancelled - cleanup */ + do { + fdd->write_q = q->next; + if (q->free) + (q->free) (q->buf); + safe_free(q); + } while ((q = fdd->write_q)); + return; + } + + fd_bytes(fd, len, FD_WRITE); if (len < 0) { if (!ignoreErrno(errno)) { status = errno == ENOSPC ? DISK_NO_SPACE_LEFT : DISK_ERROR; debug(50, 1) ("diskHandleWrite: FD %d: disk write error: %s\n", fd, xstrerror()); - if (fdd->wrt_handle == NULL) { + if (fdd->wrt_handle == NULL || status != DISK_NO_SPACE_LEFT) { /* FLUSH PENDING BUFFERS */ do { fdd->write_q = q->next; @@ -301,6 +324,9 @@ diskHandleWriteComplete(void *data, int len, int errcode) if (q != NULL) { /* q might become NULL from write failure above */ q->buf_offset += len; + if (q->buf_offset > q->len) + debug(50, 1) ("diskHandleWriteComplete: q->buf_offset > q->len (%p,%d, %d, %d FD %d)\n", + q, q->buf_offset, q->len, len, fd); assert(q->buf_offset <= q->len); if (q->buf_offset == q->len) { /* complete write */ @@ -414,7 +440,15 @@ diskHandleReadComplete(void *data, int len, int errcode) int fd = ctrlp->fd; int rc = DISK_OK; errno = errcode; + xfree(data); + + if(len == -2 && errcode == -2) { /* Read cancelled - cleanup */ + cbdataUnlock(ctrl_dat->client_data); + safe_free(ctrl_dat); + return; + } + fd_bytes(fd, len, FD_READ); if (len < 0) { if (ignoreErrno(errno)) { diff --git a/src/errorpage.cc b/src/errorpage.cc index 68fe785f0f..9cecf03ce2 100644 --- a/src/errorpage.cc +++ b/src/errorpage.cc @@ -1,6 +1,6 @@ /* - * $Id: errorpage.cc,v 1.115 1998/01/12 04:30:00 wessels Exp $ + * $Id: errorpage.cc,v 1.116 1998/02/02 21:15:00 wessels Exp $ * * DEBUG: section 4 Error Generation * AUTHOR: Duane Wessels @@ -64,7 +64,7 @@ errorInitialize(void) for (i = ERR_NONE + 1; i < ERR_MAX; i++) { snprintf(path, MAXPATHLEN, "%s/%s", Config.errorDirectory, err_type_str[i]); - fd = file_open(path, O_RDONLY, NULL, NULL); + fd = file_open(path, O_RDONLY, NULL, NULL, NULL); if (fd < 0) { debug(4, 0) ("errorInitialize: %s: %s\n", path, xstrerror()); fatal("Failed to open error text file"); diff --git a/src/fd.cc b/src/fd.cc index a033b0cdf5..8a451e3249 100644 --- a/src/fd.cc +++ b/src/fd.cc @@ -1,6 +1,6 @@ /* - * $Id: fd.cc,v 1.17 1998/01/31 05:31:57 wessels Exp $ + * $Id: fd.cc,v 1.18 1998/02/02 21:15:01 wessels Exp $ * * DEBUG: section 51 Filedescriptor Functions * AUTHOR: Duane Wessels @@ -40,12 +40,14 @@ fdUpdateBiggest(int fd, unsigned int status) return; assert(fd < Squid_MaxFD); if (fd > Biggest_FD) { - assert(status == FD_OPEN); + if (status != FD_OPEN) + debug(51,1) ("fdUpdateBiggest: status != FD_OPEN\n"); Biggest_FD = fd; return; } /* if we are here, then fd == Biggest_FD */ - assert(status == FD_CLOSE); + if (status != FD_CLOSE) + debug(51,1) ("fdUpdateBiggest: status != FD_CLOSE\n"); while (fd_table[Biggest_FD].open != FD_OPEN) Biggest_FD--; } @@ -69,6 +71,10 @@ fd_open(int fd, unsigned int type, const char *desc) { fde *F = &fd_table[fd]; assert(fd >= 0); + if(F->open != 0) { + debug(51, 1) ("WARNING: Closing open FD %4d\n", fd); + fd_close(fd); + } assert(F->open == 0); debug(51, 3) ("fd_open FD %d %s\n", fd, desc); F->type = type; diff --git a/src/filemap.cc b/src/filemap.cc index d70f30eab4..7a6fad9cc5 100644 --- a/src/filemap.cc +++ b/src/filemap.cc @@ -1,5 +1,5 @@ /* - * $Id: filemap.cc,v 1.21 1997/11/12 23:47:38 wessels Exp $ + * $Id: filemap.cc,v 1.22 1998/02/02 21:15:01 wessels Exp $ * * DEBUG: section 8 Swap File Bitmap * AUTHOR: Harvest Derived @@ -174,6 +174,8 @@ file_map_allocate(fileMap * fm, int suggestion) int word; int bit; int count; + if (suggestion > fm->max_n_files) + suggestion = 0; if (!file_map_bit_test(fm, suggestion)) { fm->last_file_number_allocated = suggestion; return file_map_bit_set(fm, suggestion); diff --git a/src/main.cc b/src/main.cc index d02d75f4d0..51486eae1b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,6 @@ /* - * $Id: main.cc,v 1.209 1998/01/31 05:31:59 wessels Exp $ + * $Id: main.cc,v 1.210 1998/02/02 21:15:03 wessels Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -445,7 +445,9 @@ mainInitialize(void) #endif if (!configured_once) { +#if !USE_ASYNC_IO unlinkdInit(); +#endif /* module initialization */ urlInitialize(); objcacheInit(); diff --git a/src/mime.cc b/src/mime.cc index 70380d9337..1e74588b4f 100644 --- a/src/mime.cc +++ b/src/mime.cc @@ -1,6 +1,6 @@ /* - * $Id: mime.cc,v 1.46 1998/01/12 04:30:41 wessels Exp $ + * $Id: mime.cc,v 1.47 1998/02/02 21:15:03 wessels Exp $ * * DEBUG: section 25 MIME Parsing * AUTHOR: Harvest Derived @@ -405,7 +405,7 @@ mimeLoadIconFile(const char *icon) if (storeGet(key)) return; snprintf(path, MAXPATHLEN, "%s/%s", Config.icons.directory, icon); - fd = file_open(path, O_RDONLY, NULL, NULL); + fd = file_open(path, O_RDONLY, NULL, NULL, NULL); if (fd < 0) { debug(25, 0) ("mimeLoadIconFile: %s: %s\n", path, xstrerror()); return; diff --git a/src/protos.h b/src/protos.h index 91a3b43c3d..6c5f9046e3 100644 --- a/src/protos.h +++ b/src/protos.h @@ -27,6 +27,7 @@ extern void aclDestroyRegexList(struct _relist *data); extern int aclMatchRegex(relist * data, const char *word); extern void aclParseRegexList(void *curlist); +#if USE_ASYNC_IO extern int aio_cancel(aio_result_t *); extern int aio_open(const char *, int, mode_t, aio_result_t *); extern int aio_read(int, char *, int, off_t, int, aio_result_t *); @@ -37,15 +38,15 @@ extern int aio_unlink(const char *, aio_result_t *); extern int aio_opendir(const char *, aio_result_t *); extern aio_result_t *aio_poll_done(void); - -extern void aioCancel(int); -extern void aioOpen(const char *, int, mode_t, AIOCB *, void *); +extern void aioCancel(int, void *); +extern void aioOpen(const char *, int, mode_t, AIOCB *, void *, void *); extern void aioClose(int); extern void aioWrite(int, char *, int, AIOCB *, void *); extern void aioRead(int, char *, int, AIOCB *, void *); -extern void aioStat(char *, struct stat *, AIOCB *, void *); +extern void aioStat(char *, struct stat *, AIOCB *, void *, void *); extern void aioUnlink(const char *, AIOCB *, void *); extern void aioCheckCallbacks(void); +#endif extern int parseConfigFile(const char *file_name); extern void intlistDestroy(intlist **); @@ -119,7 +120,7 @@ extern void _db_print(); #endif -extern int file_open(const char *path, int mode, FOCB *, void *callback_data); +extern int file_open(const char *path, int mode, FOCB *, void *callback_data, void *tag); extern void file_close(int fd); extern int file_write(int, off_t, char *, int len, DWCB *, void *, FREE *); extern int file_read(int, char *, int, off_t, DRCB *, void *); diff --git a/src/send-announce.cc b/src/send-announce.cc index 2bec2a85c7..0077a0eed3 100644 --- a/src/send-announce.cc +++ b/src/send-announce.cc @@ -1,6 +1,6 @@ /* - * $Id: send-announce.cc,v 1.45 1997/11/05 05:29:35 wessels Exp $ + * $Id: send-announce.cc,v 1.46 1998/02/02 21:15:05 wessels Exp $ * * DEBUG: section 27 Cache Announcer * AUTHOR: Duane Wessels @@ -78,7 +78,7 @@ send_announce(const ipcache_addrs * ia, void *datanotused) strcat(sndbuf, tbuf); l = strlen(sndbuf); if ((file = Config.Announce.file) != NULL) { - fd = file_open(file, O_RDONLY, NULL, NULL); + fd = file_open(file, O_RDONLY, NULL, NULL, NULL); if (fd > -1 && (n = read(fd, sndbuf + l, BUFSIZ - l - 1)) > 0) { fd_bytes(fd, n, FD_READ); l += n; diff --git a/src/stmem.cc b/src/stmem.cc index d887f45436..de48ace659 100644 --- a/src/stmem.cc +++ b/src/stmem.cc @@ -1,6 +1,6 @@ /* - * $Id: stmem.cc,v 1.55 1998/01/12 04:30:12 wessels Exp $ + * $Id: stmem.cc,v 1.56 1998/02/02 21:15:06 wessels Exp $ * * DEBUG: section 19 Memory Primitives * AUTHOR: Harvest Derived @@ -220,6 +220,10 @@ stmemCopy(const mem_hdr * mem, off_t offset, char *buf, size_t size) /* Seek our way into store */ while ((t_off + p->len) < offset) { t_off += p->len; + if(!p->next) { + debug(19,1) ("memCopy: p->next == NULL\n"); + return 0; + } assert(p->next); p = p->next; } diff --git a/src/store.cc b/src/store.cc index f92029a09c..dba64f526e 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.368 1998/02/02 07:20:55 wessels Exp $ + * $Id: store.cc,v 1.369 1998/02/02 21:15:08 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -189,6 +189,7 @@ struct storeRebuildState { int expcount; /* # objects expired */ int linecount; /* # lines parsed from cache logfile */ int clashcount; /* # swapfile clashes avoided */ + int cancelcount; /* # objects cancelled */ int dupcount; /* # duplicates purged */ int invalid; /* # bad lines */ int badflags; /* # bad e->flags */ @@ -204,7 +205,7 @@ typedef struct storeCleanList { struct storeCleanList *next; } storeCleanList; -typedef void (VCB) (void *); +typedef void (VCB) (void *, int, int); typedef struct valid_ctrl_t { struct stat *sb; @@ -253,7 +254,7 @@ static StoreEntry *storeAddDiskRestore(const cache_key *, u_num32, u_num32, int); -static void destroy_MemObject(MemObject *); +static void destroy_MemObject(StoreEntry *); static void destroy_MemObjectData(MemObject *); static void destroy_StoreEntry(StoreEntry *); static void storePurgeMem(StoreEntry *); @@ -263,15 +264,17 @@ static DWCB storeSwapOutHandle; static void storeSetPrivateKey(StoreEntry *); static EVH storeDoConvertFromLog; static EVH storeCleanup; +#if OLD_CODE static VCB storeCleanupComplete; -static void storeValidate(StoreEntry *, VCB *, void *); +#endif +static void storeValidate(StoreEntry *, VCB *, void *, void *); static AIOCB storeValidateComplete; static void storeRebuiltFromDisk(struct storeRebuildState *data); static unsigned int getKeyCounter(void); -static void storePutUnusedFileno(int fileno); +static void storePutUnusedFileno(StoreEntry *e); static int storeGetUnusedFileno(void); static void storeCheckSwapOut(StoreEntry * e); -static void storeSwapoutFileOpened(void *data, int fd); +static void storeSwapoutFileOpened(void *data, int fd, int errcode); static int storeCheckCachable(StoreEntry * e); static int storeKeepInMemory(const StoreEntry *); static SIH storeClientCopyFileOpened; @@ -286,9 +289,7 @@ static void storeEntryDump(StoreEntry * e); /* functions implementing meta data on store */ static void storeConvert(void); -static void - storeConvertFile(const cache_key *, int, int, time_t, time_t, time_t, time_t, - u_num32, u_num32, int); +static void storeConvertFile(const cache_key *, int, int, time_t, time_t, time_t, time_t, u_num32, u_num32, int); static int storeBuildMetaData(StoreEntry *, char *); static int storeGetMetaBuf(const char *, MemObject *); @@ -320,6 +321,11 @@ static int store_hash_buckets; static int store_maintain_rate; static int store_maintain_buckets; +#if OLD_CODE +/* outstanding cleanup validations */ +static int outvalid = 0; +#endif + static MemObject * new_MemObject(const char *url, const char *log_url) { @@ -349,12 +355,17 @@ new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) } static void -destroy_MemObject(MemObject * mem) +destroy_MemObject(StoreEntry *e) { + MemObject *mem = e->mem_obj; debug(20, 3) ("destroy_MemObject: destroying %p\n", mem); assert(mem->swapout.fd == -1); destroy_MemObjectData(mem); meta_data.misc -= strlen(mem->log_url); +#if USE_ASYNC_IO + while(mem->clients != NULL) + storeUnregister(e, mem->clients->callback_data); +#endif assert(mem->clients == NULL); safe_free(mem->swapout.meta_buf); memFree(MEM_HTTP_REPLY, mem->reply); @@ -371,7 +382,7 @@ destroy_StoreEntry(StoreEntry * e) debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); assert(e != NULL); if (e->mem_obj) - destroy_MemObject(e->mem_obj); + destroy_MemObject(e); storeHashDelete(e); assert(e->key == NULL); xfree(e); @@ -462,7 +473,7 @@ storePurgeMem(StoreEntry * e) debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", storeKeyText(e->key)); storeSetMemStatus(e, NOT_IN_MEMORY); - destroy_MemObject(e->mem_obj); + destroy_MemObject(e); e->mem_obj = NULL; if (e->swap_status != SWAPOUT_DONE) storeRelease(e); @@ -653,15 +664,8 @@ storeAddDiskRestore(const cache_key * key, EBIT_CLR(e->flag, RELEASE_REQUEST); EBIT_CLR(e->flag, KEY_PRIVATE); e->ping_status = PING_NONE; - if (clean) { - EBIT_SET(e->flag, ENTRY_VALIDATED); - /* Only set the file bit if we know its a valid entry */ - /* otherwise, set it in the validation procedure */ - storeDirMapBitSet(file_number); - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); - } else { - EBIT_CLR(e->flag, ENTRY_VALIDATED); - } + EBIT_CLR(e->flag, ENTRY_VALIDATED); + storeDirMapBitSet(e->swap_file_number); return e; } @@ -683,12 +687,17 @@ storeUnregister(StoreEntry * e, void *data) return 0; *S = sc->next; mem->nclients--; + sc->disk_op_in_progress = 0; if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) storeCheckSwapOut(e); if (sc->swapin_fd > -1) { commSetSelect(sc->swapin_fd, COMM_SELECT_READ, NULL, NULL, 0); file_close(sc->swapin_fd); } +#if USE_ASYNC_IO + else + aioCancel(-1, sc); +#endif if ((callback = sc->callback) != NULL) { /* callback with ssize = -1 to indicate unexpected termination */ debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", @@ -750,14 +759,19 @@ storeExpireNow(StoreEntry * e) } static void -storeSwapoutFileOpened(void *data, int fd) +storeSwapoutFileOpened(void *data, int fd, int errcode) { swapout_ctrl_t *ctrlp = data; int oldswapstatus = ctrlp->oldswapstatus; char *swapfilename = ctrlp->swapfilename; StoreEntry *e = ctrlp->e; MemObject *mem; + xfree(ctrlp); + if(fd == -2 && errcode == -2) { /* Cancelled - Clean up */ + xfree(swapfilename); + return; + } assert(e->swap_status == SWAPOUT_OPENING); if (fd < 0) { debug(20, 0) ("storeSwapoutFileOpened: Unable to open swapfile: %s\n", @@ -785,7 +799,9 @@ storeSwapOutStart(StoreEntry * e) swapout_ctrl_t *ctrlp; LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); storeLockObject(e); +#if !MONOTONIC_STORE if ((e->swap_file_number = storeGetUnusedFileno()) < 0) +#endif e->swap_file_number = storeDirMapAllocate(); storeSwapFullPath(e->swap_file_number, swapfilename); ctrlp = xmalloc(sizeof(swapout_ctrl_t)); @@ -796,7 +812,7 @@ storeSwapOutStart(StoreEntry * e) file_open(swapfilename, O_WRONLY | O_CREAT | O_TRUNC, storeSwapoutFileOpened, - ctrlp); + ctrlp, e); } static void @@ -805,13 +821,20 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) StoreEntry *e = data; MemObject *mem = e->mem_obj; debug(20, 3) ("storeSwapOutHandle: '%s', len=%d\n", storeKeyText(e->key), (int) len); - assert(mem != NULL); if (flag < 0) { debug(20, 1) ("storeSwapOutHandle: SwapOut failure (err code = %d).\n", flag); e->swap_status = SWAPOUT_NONE; if (e->swap_file_number > -1) { - storePutUnusedFileno(e->swap_file_number); +#if MONOTONIC_STORE +#if USE_ASYNC_IO + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); +#else + unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL)); +#endif +#else + storePutUnusedFileno(e); +#endif e->swap_file_number = -1; } if (flag == DISK_NO_SPACE_LEFT) { @@ -823,6 +846,14 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) storeSwapOutFileClose(e); return; } +#if USE_ASYNC_IO + if(mem == NULL) { + debug(20, 1) ("storeSwapOutHandle: mem == NULL : Cancelling swapout\n"); + return; + } +#else + assert(mem != NULL); +#endif mem->swapout.done_offset += len; if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len + mem->swapout.meta_len) { storeCheckSwapOut(e); @@ -873,7 +904,16 @@ storeCheckSwapOut(StoreEntry * e) (int) mem->swapout.queue_offset); debug(20, 3) ("storeCheckSwapOut: swapout.done_offset = %d\n", (int) mem->swapout.done_offset); +#if USE_ASYNC_IO + if(mem->inmem_hi < mem->swapout.queue_offset) { + storeAbort(e, 0); + assert(EBIT_TEST(e->flag, RELEASE_REQUEST)); + storeSwapOutFileClose(e); + return; + } +#else assert(mem->inmem_hi >= mem->swapout.queue_offset); +#endif swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); lowest_offset = storeLowestMemReaderOffset(e); debug(20, 3) ("storeCheckSwapOut: lowest_offset = %d\n", @@ -882,7 +922,8 @@ storeCheckSwapOut(StoreEntry * e) new_mem_lo = lowest_offset; if (!EBIT_TEST(e->flag, ENTRY_CACHABLE)) { - assert(EBIT_TEST(e->flag, KEY_PRIVATE)); + if(!EBIT_TEST(e->flag, KEY_PRIVATE)) + debug(20, 0) ("storeCheckSwapOut: Attempt to swap out a non-cacheable non-private object!\n"); stmemFreeDataUpto(mem->data, new_mem_lo); mem->inmem_lo = new_mem_lo; return; @@ -997,11 +1038,13 @@ storeAppendPrintf(va_alist) } /* start swapping in */ +/* callback_data will become the tag on which the stat/open can be aborted */ void storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) { swapin_ctrl_t *ctrlp; assert(e->mem_status == NOT_IN_MEMORY); +#if OLD_CODE if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { if (storeDirMapBitTest(e->swap_file_number)) { /* someone took our file while we weren't looking */ @@ -1009,9 +1052,9 @@ storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) return; } } +#endif debug(20, 3) ("storeSwapInStart: called for %08X %s \n", e->swap_file_number, e->key ? e->key : "[no key]"); - assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); assert(e->swap_file_number >= 0); assert(e->mem_obj != NULL); @@ -1019,22 +1062,23 @@ storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) ctrlp->e = e; ctrlp->callback = callback; ctrlp->callback_data = callback_data; - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) { - debug(20, 3) ("storeSwapInStart: calling storeSwapInValidateComplete GREEN\n"); - storeSwapInValidateComplete(ctrlp); - } else { - debug(20, 3) ("storeSwapInStart: calling storeValidate RED\n"); - storeValidate(e, storeSwapInValidateComplete, ctrlp); - - } + if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) + storeSwapInValidateComplete(ctrlp, 0, 0); + else + storeValidate(e, storeSwapInValidateComplete, ctrlp, callback_data); } static void -storeSwapInValidateComplete(void *data) +storeSwapInValidateComplete(void *data, int retcode, int errcode) { swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; StoreEntry *e; + + if(retcode == -2 && errcode == -2) { + xfree(ctrlp); + return; + } e = ctrlp->e; assert(e->mem_status == NOT_IN_MEMORY); if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { @@ -1044,21 +1088,33 @@ storeSwapInValidateComplete(void *data) return; } ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); - debug(20, 3) ("storeSwapInValidateComplete: Opening %s\n", ctrlp->path); - - file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp); + file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp, ctrlp->callback_data); } static void -storeSwapInFileOpened(void *data, int fd) +storeSwapInFileOpened(void *data, int fd, int errcode) { swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; StoreEntry *e = ctrlp->e; MemObject *mem = e->mem_obj; + struct stat sb; + + if(fd == -2 && errcode == -2) { + xfree(ctrlp->path); + xfree(ctrlp); + return; + } + assert(mem != NULL); assert(e->mem_status == NOT_IN_MEMORY); assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); + if (e->swap_status == SWAPOUT_DONE && (fd>=0) && fstat(fd, &sb) == 0) + if(sb.st_size == 0 || sb.st_size != e->object_len) { + debug(20,0) ("storeSwapInFileOpened: %s: Size mismatch: %d(fstat) != %d(object)\n", ctrlp->path, sb.st_size, e->object_len); + file_close(fd); + fd = -1; + } if (fd < 0) { debug(20, 0) ("storeSwapInStartComplete: Failed for '%s' (%s)\n", mem->url, ctrlp->path); @@ -1068,6 +1124,7 @@ storeSwapInFileOpened(void *data, int fd) xfree(ctrlp); return; } + debug(20, 5) ("storeSwapInStart: initialized swap file '%s' for '%s'\n", ctrlp->path, mem->url); (ctrlp->callback) (fd, ctrlp->callback_data); @@ -1155,25 +1212,36 @@ storeDoConvertFromLog(void *data) RB->invalid++; continue; } - storeSwapFullPath(sfileno, swapfile); if (x != 9) { RB->invalid++; continue; } - if (sfileno < 0) { - RB->invalid++; + timestamp = (time_t) scan1; + lastref = (time_t) scan2; + expires = (time_t) scan3; + lastmod = (time_t) scan4; + size = (off_t) scan5; + + if (size < 0) { + if((key = storeKeyScan(keytext)) == NULL) + continue; + if((e = storeGet(key)) == NULL) + continue; + if(e->lastref > lastref) + continue; + debug(20, 3) ("storeRebuildFromDisk: Cancelling: '%s'\n", keytext); + storeRelease(e); + RB->objcount--; + RB->cancelcount++; continue; } + + storeSwapFullPath(sfileno, swapfile); if (EBIT_TEST(scan7, KEY_PRIVATE)) { RB->badflags++; continue; } sfileno = storeDirProperFileno(d->dirn, sfileno); - timestamp = (time_t) scan1; - lastref = (time_t) scan2; - expires = (time_t) scan3; - lastmod = (time_t) scan4; - size = (off_t) scan5; key = storeKeyScan(keytext); if (key == NULL) { @@ -1216,7 +1284,7 @@ storeDoConvertFromLog(void *data) /* We'll assume the existing entry is valid, probably because * were in a slow rebuild and the the swap file number got taken * and the validation procedure hasn't run. */ - assert(RB->need_to_validate); + /* assert(RB->need_to_validate); */ RB->clashcount++; continue; } else if (e) { @@ -1255,63 +1323,52 @@ storeDoConvertFromLog(void *data) static void storeCleanup(void *datanotused) { - static storeCleanList *list = NULL; - storeCleanList *curr; static int bucketnum = -1; static int validnum = 0; StoreEntry *e; hash_link *link_ptr = NULL; - if (list == NULL) { - if (++bucketnum >= store_hash_buckets) { - debug(20, 1) (" Completed Validation Procedure\n"); - debug(20, 1) (" Validated %d Entries\n", validnum); - debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); - store_rebuilding = 0; - return; - } - link_ptr = hash_get_bucket(store_table, bucketnum); - for (; link_ptr; link_ptr = link_ptr->next) { - e = (StoreEntry *) link_ptr; - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) - continue; - if (EBIT_TEST(e->flag, RELEASE_REQUEST)) - continue; - curr = xcalloc(1, sizeof(storeCleanList)); - curr->key = storeKeyDup(e->key); - curr->next = list; - list = curr; - } - } - if (list == NULL) { - eventAdd("storeCleanup", storeCleanup, NULL, 0); + + if (++bucketnum >= store_hash_buckets) { + debug(20, 1) (" Completed Validation Procedure\n"); + debug(20, 1) (" Validated %d Entries\n", validnum); + debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); + store_rebuilding = 0; return; } - curr = list; - list = list->next; - e = (StoreEntry *) hash_lookup(store_table, curr->key); - if (e && !EBIT_TEST(e->flag, ENTRY_VALIDATED)) { - storeLockObject(e); - storeValidate(e, storeCleanupComplete, e); - if ((++validnum & 0xFFF) == 0) + link_ptr = hash_get_bucket(store_table, bucketnum); + for (; link_ptr; link_ptr = link_ptr->next) { + e = (StoreEntry *) link_ptr; + if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) + continue; + if (EBIT_TEST(e->flag, RELEASE_REQUEST)) + continue; + EBIT_SET(e->flag, ENTRY_VALIDATED); + /* Only set the file bit if we know its a valid entry */ + /* otherwise, set it in the validation procedure */ + storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + if ((++validnum & 0xFFFF) == 0) debug(20, 1) (" %7d Entries Validated so far.\n", validnum); assert(validnum <= memInUse(MEM_STOREENTRY)); } - storeKeyFree(curr->key); - xfree(curr); eventAdd("storeCleanup", storeCleanup, NULL, 0); } +#if OLD_CODE static void -storeCleanupComplete(void *data) +storeCleanupComplete(void *data, int retcode, int errcode) { StoreEntry *e = data; storeUnlockObject(e); + outvalid--; + if(retcode == -2 && errcode == -2) + return; if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) storeRelease(e); } +#endif static void -storeValidate(StoreEntry * e, VCB callback, void *callback_data) +storeValidate(StoreEntry * e, VCB callback, void *callback_data, void *tag) { valid_ctrl_t *ctrlp; char *path; @@ -1322,7 +1379,7 @@ storeValidate(StoreEntry * e, VCB callback, void *callback_data) assert(!EBIT_TEST(e->flag, ENTRY_VALIDATED)); if (e->swap_file_number < 0) { EBIT_CLR(e->flag, ENTRY_VALIDATED); - callback(callback_data); + callback(callback_data, 0, 0); return; } path = storeSwapFullPath(e->swap_file_number, NULL); @@ -1333,7 +1390,7 @@ storeValidate(StoreEntry * e, VCB callback, void *callback_data) ctrlp->callback = callback; ctrlp->callback_data = callback_data; #if USE_ASYNC_IO - aioStat(path, sb, storeValidateComplete, ctrlp); + aioStat(path, sb, storeValidateComplete, ctrlp, tag); #else /* When evaluating the actual arguments in a function call, the order * in which the arguments and the function expression are evaluated is @@ -1351,6 +1408,13 @@ storeValidateComplete(void *data, int retcode, int errcode) struct stat *sb = ctrlp->sb; StoreEntry *e = ctrlp->e; char *path; + + if(retcode == -2 && errcode == -2) { + xfree(sb); + xfree(ctrlp); + ctrlp->callback(ctrlp->callback_data, retcode, errcode); + return; + } if (retcode < 0 && errcode == EWOULDBLOCK) { path = storeSwapFullPath(e->swap_file_number, NULL); retcode = stat(path, sb); @@ -1359,11 +1423,10 @@ storeValidateComplete(void *data, int retcode, int errcode) EBIT_CLR(e->flag, ENTRY_VALIDATED); } else { EBIT_SET(e->flag, ENTRY_VALIDATED); - storeDirMapBitSet(e->swap_file_number); storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); } errno = errcode; - ctrlp->callback(ctrlp->callback_data); + ctrlp->callback(ctrlp->callback_data, retcode, errcode); xfree(sb); xfree(ctrlp); } @@ -1382,17 +1445,13 @@ storeRebuiltFromDisk(struct storeRebuildState *data) debug(20, 1) (" %7d With invalid flags.\n", data->badflags); debug(20, 1) (" %7d Objects loaded.\n", data->objcount); debug(20, 1) (" %7d Objects expired.\n", data->expcount); + debug(20, 1) (" %7d Objects cancelled.\n", data->cancelcount); debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); - if (data->need_to_validate && data->linecount) { - debug(20, 1) ("Beginning Validation Procedure\n"); - eventAdd("storeCleanup", storeCleanup, NULL, 0); - } else { - debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); - store_rebuilding = 0; - } + debug(20, 1) ("Beginning Validation Procedure\n"); + eventAdd("storeCleanup", storeCleanup, NULL, 0); memFree(MEM_4K_BUF, data->line_in); safe_free(data); } @@ -1493,9 +1552,18 @@ storeAbort(StoreEntry * e, int cbflag) InvokeHandlers(e); /* Do we need to close the swapout file? */ /* Not if we never started swapping out */ + /* But we may need to cancel an open/stat in progress if using ASYNC */ +#if USE_ASYNC_IO + aioCancel(-1, e); +#endif if (e->swap_file_number == -1) return; - /* not if a disk write is queued, the handler will close up */ +#if USE_ASYNC_IO + /* Need to cancel any pending ASYNC writes right now */ + if(mem->swapout.fd >= 0) + aioCancel(mem->swapout.fd, NULL); +#endif + /* but dont close if a disk write is queued, the handler will close up */ if (mem->swapout.queue_offset > mem->swapout.done_offset) return; /* we do */ @@ -1574,17 +1642,24 @@ storeMaintainSwapSpace(void *datanotused) int expired = 0; int max_scan; int max_remove; + int bigclean = 0; + int level = 3; static time_t last_warn_time = 0; eventAdd("storeMaintainSwapSpace", storeMaintainSwapSpace, NULL, 1); /* We can't delete objects while rebuilding swap */ if (store_rebuilding) return; - if (store_swap_size < store_swap_high) { - max_scan = 100; - max_remove = 10; + + if (store_swap_size > store_swap_high) + bigclean = 1; + if (store_swap_size > Config.Swap.maxSize) + bigclean = 1; + + if (bigclean) { + max_scan = 2500; + max_remove = 250; } else { - max_scan = 500; - max_remove = 50; + return; } debug(20, 3) ("storeMaintainSwapSpace\n"); for (m = all_list.tail; m; m = prev) { @@ -1592,20 +1667,28 @@ storeMaintainSwapSpace(void *datanotused) e = m->data; if (storeEntryLocked(e)) { locked++; - } else if (storeCheckExpired(e, 1)) { + continue; + } else if (bigclean) { expired++; storeRelease(e); + } else { + if (storeCheckExpired(e, 1)) { + expired++; + storeRelease(e); + } } if (expired > max_remove) break; if (++scanned > max_scan) break; } - debug(20, 3) ("storeMaintainSwapSpace stats:\n"); - debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); - debug(20, 3) (" %6d were scanned\n", scanned); - debug(20, 3) (" %6d were locked\n", locked); - debug(20, 3) (" %6d were expired\n", expired); + if(bigclean) + level = 1; + debug(20, level) ("storeMaintainSwapSpace stats:\n"); + debug(20, level) (" %6d objects\n", memInUse(MEM_STOREENTRY)); + debug(20, level) (" %6d were scanned\n", scanned); + debug(20, level) (" %6d were locked\n", locked); + debug(20, level) (" %6d were expired\n", expired); if (store_swap_size < Config.Swap.maxSize) return; if (squid_curtime - last_warn_time < 10) @@ -1630,21 +1713,37 @@ storeRelease(StoreEntry * e) storeReleaseRequest(e); return 0; } +#if USE_ASYNC_IO + aioCancel(-1, e); /* Make sure all forgotten async ops are cancelled */ +#else if (store_rebuilding) { debug(20, 2) ("storeRelease: Delaying release until store is rebuilt: '%s'\n", storeUrl(e)); storeExpireNow(e); storeSetPrivateKey(e); EBIT_SET(e->flag, RELEASE_REQUEST); + e->object_len = -(e->object_len); + storeDirSwapLog(e); + e->object_len = -(e->object_len); return 0; } +#endif storeLog(STORE_LOG_RELEASE, e); if (e->swap_file_number > -1) { - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) - storePutUnusedFileno(e->swap_file_number); +#if MONOTONIC_STORE +#if USE_ASYNC_IO + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); +#else + unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL)); +#endif +#else + storePutUnusedFileno(e); +#endif if (e->swap_status == SWAPOUT_DONE) storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); - e->swap_file_number = -1; + e->object_len = -(e->object_len); + storeDirSwapLog(e); + e->object_len = -(e->object_len); } storeSetMemStatus(e, NOT_IN_MEMORY); destroy_StoreEntry(e); @@ -1657,6 +1756,8 @@ storeEntryLocked(const StoreEntry * e) { if (e->lock_count) return 1; + if (e->swap_status == SWAPOUT_OPENING) + return 1; if (e->swap_status == SWAPOUT_WRITING) return 1; if (e->store_status == STORE_PENDING) @@ -1708,6 +1809,7 @@ storeClientListAdd(StoreEntry * e, void *data) sc->seen_offset = 0; sc->copy_offset = 0; sc->swapin_fd = -1; + sc->disk_op_in_progress = 0; sc->mem = mem; if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) sc->type = STORE_MEM_CLIENT; @@ -1729,7 +1831,7 @@ storeClientCopy(StoreEntry * e, { store_client *sc; static int recurse_detect = 0; - /*assert(e->store_status != STORE_ABORTED); */ + assert(e->store_status != STORE_ABORTED); assert(recurse_detect < 3); /* could == 1 for IMS not modified's */ debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", storeKeyText(e->key), @@ -1762,10 +1864,28 @@ storeClientCopy2(StoreEntry * e, store_client * sc) debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->key)); assert(callback != NULL); if (e->store_status == STORE_ABORTED) { +#if USE_ASYNC_IO + if(sc->disk_op_in_progress == 1) { + if(sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + sc->disk_op_in_progress = 0; + } +#endif sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, 0); } else if (e->store_status == STORE_OK && sc->copy_offset == e->object_len) { /* There is no more to send! */ +#if USE_ASYNC_IO + if(sc->disk_op_in_progress == 1) { + if(sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + sc->disk_op_in_progress = 0; + } +#endif sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, 0); } else if (e->store_status == STORE_PENDING && sc->seen_offset == mem->inmem_hi) { @@ -1775,6 +1895,15 @@ storeClientCopy2(StoreEntry * e, store_client * sc) /* What the client wants is in memory */ debug(20, 3) ("storeClientCopy2: Copying from memory\n"); sz = stmemCopy(mem->data, sc->copy_offset, sc->copy_buf, sc->copy_size); +#if USE_ASYNC_IO + if(sc->disk_op_in_progress == 1) { + if(sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + sc->disk_op_in_progress = 0; + } +#endif sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, sz); } else if (sc->swapin_fd < 0) { @@ -1782,12 +1911,22 @@ storeClientCopy2(StoreEntry * e, store_client * sc) assert(sc->type == STORE_DISK_CLIENT); /* gotta open the swapin file */ /* assert(sc->copy_offset == 0); */ - storeSwapInStart(e, storeClientCopyFileOpened, sc); + if(sc->disk_op_in_progress == 0) { + sc->disk_op_in_progress = 1; + storeSwapInStart(e, storeClientCopyFileOpened, sc); + } else { + debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); + } } else { debug(20, 3) ("storeClientCopy: reading from disk FD %d\n", sc->swapin_fd); assert(sc->type == STORE_DISK_CLIENT); - storeClientCopyFileRead(sc); + if(sc->disk_op_in_progress == 0) { + sc->disk_op_in_progress = 1; + storeClientCopyFileRead(sc); + } else { + debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); + } } --loopdetect; } @@ -1799,6 +1938,7 @@ storeClientCopyFileOpened(int fd, void *data) STCB *callback = sc->callback; if (fd < 0) { debug(20, 3) ("storeClientCopyFileOpened: failed\n"); + sc->disk_op_in_progress = 0; sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, -1); return; @@ -1826,6 +1966,8 @@ storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, voi MemObject *mem = sc->mem; STCB *callback = sc->callback; int hdr_len = 0; + assert(sc->disk_op_in_progress != 0); + sc->disk_op_in_progress = 0; assert(sc->callback != NULL); debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); if (sc->copy_offset == 0 && len > 0 && mem != NULL) { @@ -1929,7 +2071,7 @@ storeInit(void) if (strcmp((fname = Config.Log.store), "none") == 0) storelog_fd = -1; else - storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); + storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (storelog_fd < 0) debug(20, 1) ("Store logging disabled\n"); if (storeVerifyCacheDirs() < 0) { @@ -2018,11 +2160,12 @@ storeWriteCleanLogs(int reopen) cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL)); new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean")); cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean")); - safeunlink(new[dirn], 1); - safeunlink(cln[dirn], 1); + unlink(new[dirn]); + unlink(cln[dirn]); fd[dirn] = file_open(new[dirn], O_WRONLY | O_CREAT | O_TRUNC, NULL, + NULL, NULL); if (fd[dirn] < 0) { debug(50, 0) ("storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); @@ -2040,7 +2183,7 @@ storeWriteCleanLogs(int reopen) outbufs[dirn] = xcalloc(Config.cacheSwap.n_configured, CLEAN_BUF_SZ); outbuflens[dirn] = 0; } - for (m = all_list.head; m; m = m->next) { + for (m = all_list.tail; m; m = m->prev) { e = m->data; if (e->swap_file_number < 0) continue; @@ -2074,7 +2217,7 @@ storeWriteCleanLogs(int reopen) debug(20, 0) ("storeWriteCleanLogs: Current swap logfile not replaced.\n"); file_close(fd[dirn]); fd[dirn] = -1; - safeunlink(cln[dirn], 0); + unlink(cln[dirn]); continue; } outbuflens[dirn] = 0; @@ -2095,7 +2238,7 @@ storeWriteCleanLogs(int reopen) debug(20, 0) ("storeWriteCleanLogs: Current swap logfile not replaced.\n"); file_close(fd[dirn]); fd[dirn] = -1; - safeunlink(cln[dirn], 0); + unlink(cln[dirn]); continue; } } @@ -2108,8 +2251,8 @@ storeWriteCleanLogs(int reopen) file_close(fd[dirn]); fd[dirn] = -1; if (rename(new[dirn], cur[dirn]) < 0) { - debug(50, 0) ("storeWriteCleanLogs: rename failed: %s\n", - xstrerror()); + debug(50, 0) ("storeWriteCleanLogs: rename failed: %s, %s -> %s\n", + xstrerror(), new[dirn], cur[dirn]); } } storeDirCloseSwapLogs(); @@ -2124,7 +2267,7 @@ storeWriteCleanLogs(int reopen) for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { if (!store_rebuilding) file_close(file_open(cln[dirn], - O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL)); + O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL)); safe_free(cur[dirn]); safe_free(new[dirn]); safe_free(cln[dirn]); @@ -2148,7 +2291,10 @@ storePendingNClients(const StoreEntry * e) return 0; for (sc = mem->clients; sc; sc = nx) { nx = sc->next; - if (sc->callback_data == NULL) + /* Changed from callback_data to just callback. There can be no use */ + /* for callback_data without a callback, and sc->callback we know */ + /* gets reset, but not necessarily sc->callback_data */ + if (sc->callback == NULL) continue; npend++; } @@ -2194,7 +2340,7 @@ storeRotateLog(void) snprintf(to, MAXPATHLEN, "%s.%d", fname, 0); rename(fname, to); } - storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); + storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (storelog_fd < 0) { debug(50, 0) ("storeRotateLog: %s: %s\n", fname, xstrerror()); debug(20, 1) ("Store logging disabled\n"); @@ -2327,6 +2473,7 @@ storeTimestampsSet(StoreEntry * entry) #define FILENO_STACK_SIZE 128 static int fileno_stack[FILENO_STACK_SIZE]; +#if !MONOTONIC_STORE static int storeGetUnusedFileno(void) { @@ -2340,15 +2487,32 @@ storeGetUnusedFileno(void) } static void -storePutUnusedFileno(int fileno) -{ - assert(storeDirMapBitTest(fileno)); - storeDirMapBitReset(fileno); +storePutUnusedFileno(StoreEntry *e) +{ + assert(storeDirMapBitTest(e->swap_file_number)); + storeDirMapBitReset(e->swap_file_number); + /* If we're still rebuilding the swap state, then we need to avoid the */ + /* race condition where a new object gets pulled in, it expires, gets */ + /* its swapfileno added to the stack, and then that swapfileno gets */ + /* claimed by the rebuild. Must still remove the file though in any */ + /* event to avoid serving up the wrong data. This will leave us with */ + /* a URL pointing to no file at all, but that's okay since it'll fail */ + /* and get removed later anyway. */ + if(store_rebuilding) { + if(EBIT_TEST(e->flag, ENTRY_VALIDATED)) + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); + return; + } if (fileno_stack_count < FILENO_STACK_SIZE) - fileno_stack[fileno_stack_count++] = fileno; + fileno_stack[fileno_stack_count++] = e->swap_file_number; else - unlinkdUnlink(storeSwapFullPath(fileno, NULL)); +#if USE_ASYNC_IO + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); +#else + unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL)); +#endif } +#endif void storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) @@ -2441,6 +2605,10 @@ storeSwapOutFileClose(StoreEntry * e) MemObject *mem = e->mem_obj; if (mem->swapout.fd > -1) file_close(mem->swapout.fd); +#if USE_ASYNC_IO + else + aioCancel(-1, e); /* Make doubly certain pending ops are gone */ +#endif mem->swapout.fd = -1; storeUnlockObject(e); } @@ -2554,7 +2722,7 @@ storeGetNextFile(int *sfileno, int *size) snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", fullpath, entry->d_name); debug(20, 3) ("storeGetNextFile: Opening %s\n", fullfilename); - fd = file_open(fullfilename, O_RDONLY, NULL, NULL); + fd = file_open(fullfilename, O_RDONLY, NULL, NULL, NULL); continue; } #if 0 diff --git a/src/store_dir.cc b/src/store_dir.cc index 2874e32c05..ce56cc14e4 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir.cc,v 1.41 1998/02/02 07:20:57 wessels Exp $ + * $Id: store_dir.cc,v 1.42 1998/02/02 21:15:09 wessels Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -52,11 +52,19 @@ storeSwapFullPath(int fn, char *fullpath) if (!fullpath) fullpath = fullfilename; fullpath[0] = '\0'; +#if MONOTONIC_STORE + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", + Config.cacheSwap.swapDirs[dirn].path, + ((filn / Config.cacheSwap.swapDirs[dirn].l2) / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l1, + (filn / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l2, + filn); +#else snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", Config.cacheSwap.swapDirs[dirn].path, filn % Config.cacheSwap.swapDirs[dirn].l1, filn / Config.cacheSwap.swapDirs[dirn].l1 % Config.cacheSwap.swapDirs[dirn].l2, filn); +#endif return fullpath; } @@ -83,10 +91,17 @@ storeSwapSubSubDir(int fn, char *fullpath) if (!fullpath) fullpath = fullfilename; fullpath[0] = '\0'; +#if MONOTONIC_STORE snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[dirn].path, + ((filn / Config.cacheSwap.swapDirs[dirn].l2) / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l1, + (filn / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l2); +#else + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", Config.cacheSwap.swapDirs[dirn].path, filn % Config.cacheSwap.swapDirs[dirn].l1, filn / Config.cacheSwap.swapDirs[dirn].l1 % Config.cacheSwap.swapDirs[dirn].l2); +#endif return fullpath; } @@ -212,7 +227,9 @@ storeDirMapBitSet(int fn) int dirn = fn >> SWAP_DIR_SHIFT; int filn = fn & SWAP_FILE_MASK; file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn); +#if !MONOTONIC_STORE Config.cacheSwap.swapDirs[dirn].suggest++; +#endif } void @@ -221,8 +238,10 @@ storeDirMapBitReset(int fn) int dirn = fn >> SWAP_DIR_SHIFT; int filn = fn & SWAP_FILE_MASK; file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn); +#if !MONOTONIC_STORE if (fn < Config.cacheSwap.swapDirs[dirn].suggest) Config.cacheSwap.swapDirs[dirn].suggest = fn; +#endif } int @@ -231,6 +250,9 @@ storeDirMapAllocate(void) int dirn = storeMostFreeSwapDir(); SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; int filn = file_map_allocate(SD->map, SD->suggest); +#if MONOTONIC_STORE + SD->suggest = filn + 1; +#endif return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK); } @@ -258,12 +280,15 @@ storeDirSwapLog(const StoreEntry * e) { LOCAL_ARRAY(char, logmsg, MAX_URL << 1); int dirn; + assert(e->swap_file_number >= 0); dirn = e->swap_file_number >> SWAP_DIR_SHIFT; assert(dirn < Config.cacheSwap.n_configured); +#if !USE_ASYNC_IO assert(!EBIT_TEST(e->flag, KEY_PRIVATE)); +#endif /* Note this printf format appears in storeWriteCleanLog() too */ - snprintf(logmsg, MAX_URL << 1, "%08x %08x %08x %08x %08x %9d %6d %08x %s\n", + snprintf(logmsg, MAX_URL << 1, "%08x %08x %08x %08x %08x %d %6d %08x %s\n", (int) e->swap_file_number, (int) e->timestamp, (int) e->lastref, @@ -311,7 +336,7 @@ storeDirOpenSwapLogs(void) for (i = 0; i < Config.cacheSwap.n_configured; i++) { SD = &Config.cacheSwap.swapDirs[i]; path = storeDirSwapLogFile(i, NULL); - fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL); + fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (fd < 0) { debug(50, 1) ("%s: %s\n", path, xstrerror()); fatal("storeDirOpenSwapLogs: Failed to open swap log."); @@ -358,7 +383,7 @@ storeDirOpenTmpSwapLog(int dirn, int *clean_flag) if (SD->swaplog_fd >= 0) file_close(SD->swaplog_fd); /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL); + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL); if (fd < 0) { debug(50, 1) ("%s: %s\n", new_path, xstrerror()); fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); @@ -396,7 +421,7 @@ storeDirCloseTmpSwapLog(int dirn) fatal("storeDirCloseTmpSwapLog: rename failed"); } file_close(SD->swaplog_fd); - fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL); + fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL, NULL); if (fd < 0) { debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); fatal("storeDirCloseTmpSwapLog: Failed to open swap log."); diff --git a/src/structs.h b/src/structs.h index d39662c980..57f1e9c9bf 100644 --- a/src/structs.h +++ b/src/structs.h @@ -789,6 +789,7 @@ struct _store_client { void *callback_data; MemObject *mem; /* ptr to the parent structure, argh! */ int swapin_fd; + int disk_op_in_progress; struct _store_client *next; }; diff --git a/src/tools.cc b/src/tools.cc index b31fff4b4f..a63dbc8e2e 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -1,6 +1,6 @@ /* - * $Id: tools.cc,v 1.142 1998/01/31 05:32:09 wessels Exp $ + * $Id: tools.cc,v 1.143 1998/02/02 21:15:10 wessels Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -384,7 +384,9 @@ normal_shutdown(void) leave_suid(); } releaseServerSockets(); +#if !USE_ASYNC_IO unlinkdClose(); +#endif storeWriteCleanLogs(0); PrintRusage(); dumpMallocStats(); @@ -629,7 +631,7 @@ writePidFile(void) return; enter_suid(); old_umask = umask(022); - fd = file_open(f, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL); + fd = file_open(f, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL); umask(old_umask); leave_suid(); if (fd < 0) { diff --git a/src/typedefs.h b/src/typedefs.h index 444072567b..ed02794c05 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -77,11 +77,12 @@ typedef struct _dlink_node dlink_node; typedef struct _dlink_list dlink_list; typedef struct _StatCounters StatCounters; +/* define AIOCB even without USE_ASYNC_IO */ typedef void AIOCB(void *, int aio_return, int aio_errno); typedef void CWCB(int fd, char *, size_t size, int flag, void *data); typedef void CNCB(int fd, int status, void *); typedef void FREE(void *); -typedef void FOCB(void *, int fd); +typedef void FOCB(void *, int fd, int errcode); typedef void EVH(void *); typedef void PF(int, void *); typedef void DRCB(int fd, const char *buf, int size, int errflag, void *data); diff --git a/src/unlinkd.cc b/src/unlinkd.cc index e9180d854e..897eb9c9c2 100644 --- a/src/unlinkd.cc +++ b/src/unlinkd.cc @@ -1,5 +1,5 @@ /* - * $Id: unlinkd.cc,v 1.14 1998/02/02 07:20:58 wessels Exp $ + * $Id: unlinkd.cc,v 1.15 1998/02/02 21:15:12 wessels Exp $ * * DEBUG: section 43 Unlink Daemon * AUTHOR: Duane Wessels @@ -74,6 +74,8 @@ main(int argc, char *argv[]) #include "squid.h" +#if !USE_ASYNC_IO + static int unlinkd_fd = -1; static int unlinkdCreate(void); @@ -151,4 +153,5 @@ unlinkdInit(void) debug(43, 0) ("Unlinkd pipe opened on FD %d\n", unlinkd_fd); } +#endif /* !USE_ASYNC_IO */ #endif /* ndef UNLINK_DAEMON */ diff --git a/src/useragent.cc b/src/useragent.cc index e0b86fc880..26cde5f1be 100644 --- a/src/useragent.cc +++ b/src/useragent.cc @@ -1,6 +1,6 @@ /* - * $Id: useragent.cc,v 1.12 1997/10/17 00:00:50 wessels Exp $ + * $Id: useragent.cc,v 1.13 1998/02/02 21:15:12 wessels Exp $ * * DEBUG: section 40 User-Agent logging * AUTHOR: Joe Ramey @@ -46,7 +46,7 @@ useragentOpenLog(void) cache_useragent_log = NULL; } if (fname && strcmp(fname, "none") != 0) { - log_fd = file_open(fname, O_WRONLY | O_CREAT | O_APPEND, NULL, NULL); + log_fd = file_open(fname, O_WRONLY | O_CREAT | O_APPEND, NULL, NULL, NULL); if (log_fd < 0) { debug(50, 0) ("useragentOpenLog: %s: %s\n", fname, xstrerror()); } else if ((cache_useragent_log = fdopen(log_fd, "a")) == NULL) {