From add2192dfeb5b40768d31ba3b9b05151263af5b5 Mon Sep 17 00:00:00 2001 From: adrian <> Date: Tue, 26 Feb 2002 22:48:09 +0000 Subject: [PATCH] Bring across my first phase of commloops development. This focused on turning storeClientCopy() into a stream-type callback, trying to act like the stream storeAppend() is for the server side. * storeClientCopy() has lost the seen_offset parameter * storeClientCopy*() track the last copy offset and size, so the code can _enforce_ that the modules using it have been converted into stream routines properly * all the modules using storeClientCopy() have been converted. I'm sure there are remaining places where the stream enforce will be triggered - these are the evil places noone speaks of, and will probably be exorcised in the third round of commloops (which will probably be a modio exercise.) --- Makefile.in | 2 +- configure | 2 +- doc/Makefile.in | 2 +- errors/Makefile.in | 2 +- helpers/basic_auth/LDAP/Makefile.in | 2 +- helpers/basic_auth/MSNT/Makefile.in | 2 +- helpers/basic_auth/Makefile.in | 2 +- helpers/basic_auth/NCSA/Makefile.in | 2 +- helpers/basic_auth/PAM/Makefile.in | 2 +- helpers/basic_auth/SMB/Makefile.in | 2 +- helpers/basic_auth/YP/Makefile.in | 2 +- helpers/basic_auth/getpwnam/Makefile.in | 2 +- .../basic_auth/multi-domain-NTLM/Makefile.in | 2 +- helpers/digest_auth/Makefile.in | 2 +- helpers/digest_auth/password/Makefile.in | 2 +- helpers/ntlm_auth/Makefile.in | 2 +- helpers/ntlm_auth/SMB/Makefile.in | 2 +- helpers/ntlm_auth/fakeauth/Makefile.in | 2 +- helpers/ntlm_auth/no_check/Makefile.in | 2 +- icons/Makefile.in | 2 +- lib/Makefile.in | 2 +- src/Makefile.in | 2 +- src/asn.cc | 80 ++- src/auth/Makefile.in | 2 +- src/client_side.cc | 549 +++--------------- src/defines.h | 6 +- src/enums.h | 15 +- src/fs/Makefile.in | 2 +- src/fs/diskd/Makefile.in | 2 +- src/http.cc | 30 +- src/mem.cc | 3 +- src/net_db.cc | 130 ++++- src/peer_digest.cc | 213 +++++-- src/protos.h | 5 +- src/repl/Makefile.in | 2 +- src/stat.cc | 4 +- src/store_client.cc | 46 +- src/structs.h | 14 +- src/urn.cc | 53 +- 39 files changed, 513 insertions(+), 687 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8a126e89ea..0e339c07e2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ # -# $Id: Makefile.in,v 1.7 2001/12/27 13:11:57 hno Exp $ +# $Id: Makefile.in,v 1.8 2002/02/26 15:48:09 adrian Exp $ # SHELL = @SHELL@ diff --git a/configure b/configure index 70a55c2fdb..3dce74d1ed 100755 --- a/configure +++ b/configure @@ -1560,7 +1560,7 @@ else if { (eval echo configure:1561: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; }; then for file in conftest.*; do case $file in - *.c | *.o | *.obj) ;; + *.c | *.C | *.o | *.obj) ;; *) ac_cv_exeext=`echo $file | sed -e s/conftest//` ;; esac done diff --git a/doc/Makefile.in b/doc/Makefile.in index 855dfa01d4..930af4c0f2 100644 --- a/doc/Makefile.in +++ b/doc/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.2 2001/12/27 02:22:35 hno Exp $ +# $Id: Makefile.in,v 1.3 2002/02/26 15:48:10 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/errors/Makefile.in b/errors/Makefile.in index 70be00f772..67fab1b505 100644 --- a/errors/Makefile.in +++ b/errors/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ # -# $Id: Makefile.in,v 1.15 2001/11/30 15:35:45 hno Exp $ +# $Id: Makefile.in,v 1.16 2002/02/26 15:48:11 adrian Exp $ # SHELL = @SHELL@ diff --git a/helpers/basic_auth/LDAP/Makefile.in b/helpers/basic_auth/LDAP/Makefile.in index 045f14dcf7..7ff0367aa9 100644 --- a/helpers/basic_auth/LDAP/Makefile.in +++ b/helpers/basic_auth/LDAP/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid LDAP authentication helper # -# $Id: Makefile.in,v 1.10 2002/01/08 16:24:15 hno Exp $ +# $Id: Makefile.in,v 1.11 2002/02/26 15:48:18 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/MSNT/Makefile.in b/helpers/basic_auth/MSNT/Makefile.in index 95f6f10621..cbe53dec0b 100644 --- a/helpers/basic_auth/MSNT/Makefile.in +++ b/helpers/basic_auth/MSNT/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/12/27 00:23:32 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:19 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/Makefile.in b/helpers/basic_auth/Makefile.in index 8d3897dd48..03cac42423 100644 --- a/helpers/basic_auth/Makefile.in +++ b/helpers/basic_auth/Makefile.in @@ -15,7 +15,7 @@ # Makefile for storage modules in the Squid Object Cache server # -# $Id: Makefile.in,v 1.7 2001/11/29 11:17:03 hno Exp $ +# $Id: Makefile.in,v 1.8 2002/02/26 15:48:18 adrian Exp $ # SHELL = @SHELL@ diff --git a/helpers/basic_auth/NCSA/Makefile.in b/helpers/basic_auth/NCSA/Makefile.in index 767524ff23..80af054c85 100644 --- a/helpers/basic_auth/NCSA/Makefile.in +++ b/helpers/basic_auth/NCSA/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:06 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:20 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/PAM/Makefile.in b/helpers/basic_auth/PAM/Makefile.in index 8ab551c4b6..7c5d6c246d 100644 --- a/helpers/basic_auth/PAM/Makefile.in +++ b/helpers/basic_auth/PAM/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid PAM authentication helper # -# $Id: Makefile.in,v 1.9 2002/01/08 16:24:21 hno Exp $ +# $Id: Makefile.in,v 1.10 2002/02/26 15:48:20 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/SMB/Makefile.in b/helpers/basic_auth/SMB/Makefile.in index a6bb0c90b8..97cc97ea63 100644 --- a/helpers/basic_auth/SMB/Makefile.in +++ b/helpers/basic_auth/SMB/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/12/27 00:23:33 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:21 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/YP/Makefile.in b/helpers/basic_auth/YP/Makefile.in index 52615dd592..f0b1a35e0c 100644 --- a/helpers/basic_auth/YP/Makefile.in +++ b/helpers/basic_auth/YP/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.10 2001/12/27 00:23:33 hno Exp $ +# $Id: Makefile.in,v 1.11 2002/02/26 15:48:22 adrian Exp $ # # diff --git a/helpers/basic_auth/getpwnam/Makefile.in b/helpers/basic_auth/getpwnam/Makefile.in index a94282bc7e..3ae4fd09ad 100644 --- a/helpers/basic_auth/getpwnam/Makefile.in +++ b/helpers/basic_auth/getpwnam/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:10 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:23 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/basic_auth/multi-domain-NTLM/Makefile.in b/helpers/basic_auth/multi-domain-NTLM/Makefile.in index facbd64046..59d026e488 100644 --- a/helpers/basic_auth/multi-domain-NTLM/Makefile.in +++ b/helpers/basic_auth/multi-domain-NTLM/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.6 2001/12/27 00:23:33 hno Exp $ +# $Id: Makefile.in,v 1.7 2002/02/26 15:48:24 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/digest_auth/Makefile.in b/helpers/digest_auth/Makefile.in index 688827709c..c7a5cc86b3 100644 --- a/helpers/digest_auth/Makefile.in +++ b/helpers/digest_auth/Makefile.in @@ -15,7 +15,7 @@ # Makefile for digest auth helpers in the Squid Object Cache server # -# $Id: Makefile.in,v 1.6 2001/11/29 11:17:13 hno Exp $ +# $Id: Makefile.in,v 1.7 2002/02/26 15:48:24 adrian Exp $ # SHELL = @SHELL@ diff --git a/helpers/digest_auth/password/Makefile.in b/helpers/digest_auth/password/Makefile.in index afe59358aa..59073c3363 100644 --- a/helpers/digest_auth/password/Makefile.in +++ b/helpers/digest_auth/password/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:18 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:25 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/ntlm_auth/Makefile.in b/helpers/ntlm_auth/Makefile.in index 83db37343d..f418f6a479 100644 --- a/helpers/ntlm_auth/Makefile.in +++ b/helpers/ntlm_auth/Makefile.in @@ -15,7 +15,7 @@ # Makefile for storage modules in the Squid Object Cache server # -# $Id: Makefile.in,v 1.7 2001/11/30 10:07:33 hno Exp $ +# $Id: Makefile.in,v 1.8 2002/02/26 15:48:26 adrian Exp $ # SHELL = @SHELL@ diff --git a/helpers/ntlm_auth/SMB/Makefile.in b/helpers/ntlm_auth/SMB/Makefile.in index 69bcd33326..9c7657daa3 100644 --- a/helpers/ntlm_auth/SMB/Makefile.in +++ b/helpers/ntlm_auth/SMB/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:24 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:26 adrian Exp $ # SHELL = @SHELL@ diff --git a/helpers/ntlm_auth/fakeauth/Makefile.in b/helpers/ntlm_auth/fakeauth/Makefile.in index 1d41d7a383..5add29227c 100644 --- a/helpers/ntlm_auth/fakeauth/Makefile.in +++ b/helpers/ntlm_auth/fakeauth/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:27 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:27 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/helpers/ntlm_auth/no_check/Makefile.in b/helpers/ntlm_auth/no_check/Makefile.in index 302d5121df..084f828452 100644 --- a/helpers/ntlm_auth/no_check/Makefile.in +++ b/helpers/ntlm_auth/no_check/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.9 2001/12/27 00:23:34 hno Exp $ +# $Id: Makefile.in,v 1.10 2002/02/26 15:48:28 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/icons/Makefile.in b/icons/Makefile.in index 4d47a1b880..9d0ad81a41 100644 --- a/icons/Makefile.in +++ b/icons/Makefile.in @@ -13,7 +13,7 @@ @SET_MAKE@ -# $Id: Makefile.in,v 1.21 2001/11/30 15:35:46 hno Exp $ +# $Id: Makefile.in,v 1.22 2002/02/26 15:48:11 adrian Exp $ # SHELL = @SHELL@ diff --git a/lib/Makefile.in b/lib/Makefile.in index 12aee50407..94dad7f5c7 100644 --- a/lib/Makefile.in +++ b/lib/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ # -# $Id: Makefile.in,v 1.57 2001/11/29 11:16:53 hno Exp $ +# $Id: Makefile.in,v 1.58 2002/02/26 15:48:12 adrian Exp $ # SHELL = @SHELL@ diff --git a/src/Makefile.in b/src/Makefile.in index 155945c8a3..a83f7e0108 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.230 2002/01/01 09:47:48 adrian Exp $ +# $Id: Makefile.in,v 1.231 2002/02/26 15:48:13 adrian Exp $ # # Uncomment and customize the following to suit your needs: # diff --git a/src/asn.cc b/src/asn.cc index dd1a7fc063..0462fb0b60 100644 --- a/src/asn.cc +++ b/src/asn.cc @@ -1,6 +1,6 @@ /* - * $Id: asn.cc,v 1.78 2001/11/13 19:24:35 hno Exp $ + * $Id: asn.cc,v 1.79 2002/02/26 15:48:13 adrian Exp $ * * DEBUG: section 53 AS Number handling * AUTHOR: Duane Wessels, Kostas Anagnostakis @@ -37,6 +37,7 @@ #include "radix.h" #define WHOIS_PORT 43 +#define AS_REQBUF_SZ 4096 /* BEGIN of definitions for radix tree entries */ @@ -67,8 +68,9 @@ struct _ASState { store_client *sc; request_t *request; int as_number; - off_t seen; off_t offset; + int reqofs; + char reqbuf[AS_REQBUF_SZ]; }; typedef struct _ASState ASState; @@ -205,49 +207,55 @@ asnCacheStart(int as) asState->sc = storeClientListAdd(e, asState); } asState->entry = e; - asState->seen = 0; asState->offset = 0; + asState->reqofs = 0; storeClientCopy(asState->sc, e, - asState->seen, asState->offset, - 4096, - memAllocate(MEM_4K_BUF), + AS_REQBUF_SZ, + asState->reqbuf, asHandleReply, asState); } static void -asHandleReply(void *data, char *buf, ssize_t size) +asHandleReply(void *data, char *unused_buf, ssize_t retsize) { ASState *asState = data; StoreEntry *e = asState->entry; char *s; char *t; - debug(53, 3) ("asHandleReply: Called with size=%ld\n", (long int) size); + char *buf = asState->reqbuf; + int leftoversz = -1; + + debug(53, 3) ("asHandleReply: Called with size=%d\n", (int)retsize); + debug(53, 3) ("asHandleReply: buffer='%s'\n", buf); + + /* First figure out whether we should abort the request */ if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { - memFree(buf, MEM_4K_BUF); asStateFree(asState); return; } - if (size == 0 && e->mem_obj->inmem_hi > 0) { - memFree(buf, MEM_4K_BUF); + if (retsize == 0 && e->mem_obj->inmem_hi > 0) { asStateFree(asState); return; - } else if (size < 0) { - debug(53, 1) ("asHandleReply: Called with size=%ld\n", (long int) size); - memFree(buf, MEM_4K_BUF); + } else if (retsize < 0) { + debug(53, 1) ("asHandleReply: Called with size=%d\n", retsize); asStateFree(asState); return; } else if (HTTP_OK != e->mem_obj->reply->sline.status) { debug(53, 1) ("WARNING: AS %d whois request failed\n", asState->as_number); - memFree(buf, MEM_4K_BUF); asStateFree(asState); return; } + + /* + * Next, attempt to parse our request + * Remembering that the actual buffer size is retsize + reqofs! + */ s = buf; - while (s - buf < size && *s != '\0') { + while (s - buf < (retsize + asState->reqofs) && *s != '\0') { while (*s && xisspace(*s)) s++; for (t = s; *t; t++) { @@ -263,33 +271,47 @@ asHandleReply(void *data, char *buf, ssize_t size) asnAddNet(s, asState->as_number); s = t + 1; } - asState->seen = asState->offset + size; - asState->offset += (s - buf); - debug(53, 3) ("asState->seen = %ld, asState->offset = %ld\n", - (long int) asState->seen, (long int) asState->offset); + + /* + * Next, grab the end of the 'valid data' in the buffer, and figure + * out how much data is left in our buffer, which we need to keep + * around for the next request + */ + leftoversz = (asState->reqofs + retsize) - (s - buf); + assert(leftoversz >= 0); + + /* + * Next, copy the left over data, from s to s + leftoversz to the + * beginning of the buffer + */ + xmemmove(buf, s, leftoversz); + + /* + * Next, update our offset and reqofs, and kick off a copy if required + */ + asState->offset += retsize; + asState->reqofs = leftoversz; + debug(53, 3) ("asState->offset = %ld\n",(long int) asState->offset); if (e->store_status == STORE_PENDING) { debug(53, 3) ("asHandleReply: store_status == STORE_PENDING: %s\n", storeUrl(e)); storeClientCopy(asState->sc, e, - asState->seen, asState->offset, - 4096, - buf, + AS_REQBUF_SZ - asState->reqofs, + asState->reqbuf + asState->reqofs, asHandleReply, asState); - } else if (asState->seen < e->mem_obj->inmem_hi) { - debug(53, 3) ("asHandleReply: asState->seen < e->mem_obj->inmem_hi %s\n", storeUrl(e)); + } else if (asState->offset < e->mem_obj->inmem_hi) { + debug(53, 3) ("asHandleReply: asState->offset < e->mem_obj->inmem_hi %s\n", storeUrl(e)); storeClientCopy(asState->sc, e, - asState->seen, asState->offset, - 4096, - buf, + AS_REQBUF_SZ - asState->reqofs, + asState->reqbuf + asState->reqofs, asHandleReply, asState); } else { debug(53, 3) ("asHandleReply: Done: %s\n", storeUrl(e)); - memFree(buf, MEM_4K_BUF); asStateFree(asState); } } diff --git a/src/auth/Makefile.in b/src/auth/Makefile.in index 9fae82074f..a7ab5cbaaf 100644 --- a/src/auth/Makefile.in +++ b/src/auth/Makefile.in @@ -15,7 +15,7 @@ # Makefile for authentication modules in the Squid Object Cache server # -# $Id: Makefile.in,v 1.6 2001/11/29 11:17:00 hno Exp $ +# $Id: Makefile.in,v 1.7 2002/02/26 15:48:18 adrian Exp $ # SHELL = @SHELL@ diff --git a/src/client_side.cc b/src/client_side.cc index 83e48adb08..3ad7bebe7b 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.563 2002/02/14 00:59:50 hno Exp $ + * $Id: client_side.cc,v 1.564 2002/02/26 15:48:13 adrian Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -103,8 +103,6 @@ static int clientOnlyIfCached(clientHttpRequest * http); static STCB clientSendMoreData; static STCB clientCacheHit; static void clientSetKeepaliveFlag(clientHttpRequest *); -static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); -static void clientPackTermBound(String boundary, MemBuf * mb); static void clientInterpretRequestHeaders(clientHttpRequest *); static void clientProcessRequest(clientHttpRequest *); static void clientProcessExpired(void *data); @@ -208,8 +206,13 @@ clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) #if DELAY_POOLS delaySetStoreClient(h->sc, delayClient(h->request)); #endif - storeClientCopy(h->sc, e, 0, 0, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); + h->reqofs = 0; + h->reqsize = 0; + /* I don't think this is actually needed! -- adrian */ + /* h->reqbuf = h->norm_reqbuf; */ + assert(h->reqbuf == h->norm_reqbuf); + storeClientCopy(h->sc, e, 0, HTTP_REQBUF_SZ, h->reqbuf, + clientSendMoreData, h); return e; } @@ -373,6 +376,9 @@ clientProcessExpired(void *data) http->request->flags.refresh = 1; http->old_entry = http->entry; http->old_sc = http->sc; + http->old_reqsize = http->reqsize; + http->old_reqofs = http->reqofs; + http->reqbuf = http->ims_reqbuf; /* * Assert that 'http' is already a client of old_entry. If * it is not, then the beginning of the object data might get @@ -397,11 +403,11 @@ clientProcessExpired(void *data) /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); + http->reqofs = 0; storeClientCopy(http->sc, entry, http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + HTTP_REQBUF_SZ, + http->reqbuf, clientHandleIMSReply, http); } @@ -453,17 +459,16 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) const char *url = storeUrl(entry); int unlink_request = 0; StoreEntry *oldentry; - int recopy = 1; http_status status; debug(33, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size); if (entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } + /* update size of the request */ + http->reqsize = size + http->reqofs; mem = entry->mem_obj; status = mem->reply->sline.status; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { @@ -475,9 +480,12 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) storeUnlockObject(entry); entry = http->entry = http->old_entry; http->sc = http->old_sc; + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; } else if (STORE_PENDING == entry->store_status && 0 == status) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); - if (size >= CLIENT_SOCK_SZ) { + if (size + http->reqofs >= HTTP_REQBUF_SZ) { /* will not get any bigger than that */ debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); /* use old entry, this repeats the code abovez */ @@ -486,13 +494,16 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) storeUnlockObject(entry); entry = http->entry = http->old_entry; http->sc = http->old_sc; + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; /* continue */ } else { + http->reqofs += size; storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ - http->reqofs, + http->reqbuf + http->reqofs, clientHandleIMSReply, http); return; @@ -522,6 +533,9 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) requestUnlink(entry->mem_obj->request); entry->mem_obj->request = NULL; } + http->reqbuf = http->norm_reqbuf; + http->reqofs = http->old_reqofs; + http->reqsize = http->old_reqsize; } else { /* the client can handle this reply, whatever it is */ http->log_type = LOG_TCP_REFRESH_MISS; @@ -533,22 +547,14 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) } storeUnregister(http->old_sc, http->old_entry, http); storeUnlockObject(http->old_entry); - recopy = 0; } http->old_entry = NULL; /* done with old_entry */ http->old_sc = NULL; + http->old_reqofs = 0; + http->old_reqsize = 0; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); - if (recopy) { - storeClientCopy(http->sc, entry, - http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - buf, - clientSendMoreData, - http); - } else { - clientSendMoreData(data, buf, size); - } + + clientSendMoreData(data, http->reqbuf, http->reqsize); } int @@ -620,11 +626,11 @@ clientPurgeRequest(clientHttpRequest * http) http->entry->mem_obj->method = http->request->method; http->sc = storeClientListAdd(http->entry, http); http->log_type = LOG_TCP_HIT; + http->reqofs = 0; storeClientCopy(http->sc, http->entry, http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + HTTP_REQBUF_SZ, + http->reqbuf, clientCacheHit, http); return; @@ -819,7 +825,6 @@ httpRequestFree(void *data) safe_free(http->al.headers.reply); safe_free(http->al.cache.authuser); safe_free(http->redirect.location); - stringClean(&http->range_iter.boundary); if ((e = http->entry)) { http->entry = NULL; storeUnregister(http->sc, e, http); @@ -935,9 +940,17 @@ clientInterpretRequestHeaders(clientHttpRequest * http) } /* ignore range header in non-GETs */ if (request->method == METHOD_GET) { - request->range = httpHeaderGetRange(req_hdr); - if (request->range) - request->flags.range = 1; + /* + * Since we're not doing ranges atm, just set the flag if + * the header exists, and then free the range header info + * -- adrian + */ + request->range = httpHeaderGetRange(req_hdr); + if (request->range) { + request->flags.range = 1; + httpHdrRangeDestroy(request->range); + request->range = NULL; + } } if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) request->flags.auth = 1; @@ -1114,152 +1127,6 @@ isTcpHit(log_type code) return 0; } -/* - * returns true if If-Range specs match reply, false otherwise - */ -static int -clientIfRangeMatch(clientHttpRequest * http, HttpReply * rep) -{ - const TimeOrTag spec = httpHeaderGetTimeOrTag(&http->request->header, HDR_IF_RANGE); - /* check for parsing falure */ - if (!spec.valid) - return 0; - /* got an ETag? */ - if (spec.tag.str) { - ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); - debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", - spec.tag.str, rep_tag.str ? rep_tag.str : ""); - if (!rep_tag.str) - return 0; /* entity has no etag to compare with! */ - if (spec.tag.weak || rep_tag.weak) { - debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", - spec.tag.str, rep_tag.str); - return 0; /* must use strong validator for sub-range requests */ - } - return etagIsEqual(&rep_tag, &spec.tag); - } - /* got modification time? */ - if (spec.time >= 0) { - return http->entry->lastmod <= spec.time; - } - assert(0); /* should not happen */ - return 0; -} - -/* returns expected content length for multi-range replies - * note: assumes that httpHdrRangeCanonize has already been called - * warning: assumes that HTTP headers for individual ranges at the - * time of the actuall assembly will be exactly the same as - * the headers when clientMRangeCLen() is called */ -static int -clientMRangeCLen(clientHttpRequest * http) -{ - int clen = 0; - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec; - MemBuf mb; - - assert(http->entry->mem_obj); - - memBufDefInit(&mb); - while ((spec = httpHdrRangeGetSpec(http->request->range, &pos))) { - - /* account for headers for this range */ - memBufReset(&mb); - clientPackRangeHdr(http->entry->mem_obj->reply, - spec, http->range_iter.boundary, &mb); - clen += mb.size; - - /* account for range content */ - clen += spec->length; - - debug(33, 6) ("clientMRangeCLen: (clen += %ld + %ld) == %d\n", - (long int) mb.size, (long int) spec->length, clen); - } - /* account for the terminating boundary */ - memBufReset(&mb); - clientPackTermBound(http->range_iter.boundary, &mb); - clen += mb.size; - - memBufClean(&mb); - return clen; -} - -/* adds appropriate Range headers if needed */ -static void -clientBuildRangeHeader(clientHttpRequest * http, HttpReply * rep) -{ - HttpHeader *hdr = rep ? &rep->header : 0; - const char *range_err = NULL; - request_t *request = http->request; - int is_hit = isTcpHit(http->log_type); - assert(request->range); - /* check if we still want to do ranges */ - if (!rep) - range_err = "no [parse-able] reply"; - else if (rep->sline.status != HTTP_OK) - range_err = "wrong status code"; - else if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) - range_err = "origin server does ranges"; - else if (rep->content_length < 0) - range_err = "unknown length"; - else if (rep->content_length != http->entry->mem_obj->reply->content_length) - range_err = "INCONSISTENT length"; /* a bug? */ - else if (httpHeaderHas(&http->request->header, HDR_IF_RANGE) && !clientIfRangeMatch(http, rep)) - range_err = "If-Range match failed"; - else if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) - range_err = "canonization failed"; - else if (httpHdrRangeIsComplex(http->request->range)) - range_err = "too complex range header"; - else if (!request->flags.cachable) /* from we_do_ranges in http.c */ - range_err = "non-cachable request"; - else if (!is_hit && httpHdrRangeOffsetLimit(http->request->range)) - range_err = "range outside range_offset_limit"; - /* get rid of our range specs on error */ - if (range_err) { - debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); - httpHdrRangeDestroy(http->request->range); - http->request->range = NULL; - } else { - const int spec_count = http->request->range->specs.count; - int actual_clen = -1; - - debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n", - spec_count, rep->content_length); - assert(spec_count > 0); - /* ETags should not be returned with Partial Content replies? */ - httpHeaderDelById(hdr, HDR_ETAG); - /* append appropriate header(s) */ - if (spec_count == 1) { - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec = httpHdrRangeGetSpec(http->request->range, &pos); - assert(spec); - /* append Content-Range */ - httpHeaderAddContRange(hdr, *spec, rep->content_length); - /* set new Content-Length to the actual number of bytes - * transmitted in the message-body */ - actual_clen = spec->length; - } else { - /* multipart! */ - /* generate boundary string */ - http->range_iter.boundary = httpHdrRangeBoundaryStr(http); - /* delete old Content-Type, add ours */ - httpHeaderDelById(hdr, HDR_CONTENT_TYPE); - httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, - "multipart/byteranges; boundary=\"%s\"", - strBuf(http->range_iter.boundary)); - /* Content-Length is not required in multipart responses - * but it is always nice to have one */ - actual_clen = clientMRangeCLen(http); - } - - /* replace Content-Length header */ - assert(actual_clen >= 0); - httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); - httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen); - debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen); - } -} /* * filters out unwanted entries from original reply header @@ -1304,9 +1171,6 @@ clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) httpHeaderDelById(hdr, HDR_CONNECTION); stringClean(&strConnection); } - /* Handle Ranges */ - if (request->range) - clientBuildRangeHeader(http, rep); /* * Add a estimated Age header on cache hits. */ @@ -1379,19 +1243,10 @@ clientBuildReply(clientHttpRequest * http, const char *buf, size_t size) httpBuildVersion(&rep->sline.version, 1, 0); /* do header conversions */ clientBuildReplyHeader(http, rep); - /* if we do ranges, change status to "Partial Content" */ - if (http->request->range) - httpStatusLineSet(&rep->sline, rep->sline.version, - HTTP_PARTIAL_CONTENT, NULL); } else { /* parsing failure, get rid of the invalid reply */ httpReplyDestroy(rep); rep = NULL; - /* if we were going to do ranges, backoff */ - if (http->request->range) { - /* this will fail and destroy request->range */ - clientBuildRangeHeader(http, rep); - } } return rep; } @@ -1412,12 +1267,10 @@ clientCacheHit(void *data, char *buf, ssize_t size) request_t *r = http->request; debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); if (http->entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: request aborted\n"); return; } else if (size < 0) { /* swap in failure */ - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; if ((e = http->entry)) { @@ -1432,24 +1285,25 @@ clientCacheHit(void *data, char *buf, ssize_t size) assert(size > 0); mem = e->mem_obj; assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); + /* update size of the request */ + http->reqsize = size + http->reqofs; if (mem->reply->sline.status == 0) { /* * we don't have full reply headers yet; either wait for more or * punt to clientProcessMiss. */ if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); - } else if (size == CLIENT_SOCK_SZ && http->out.offset == 0) { - memFree(buf, MEM_CLIENT_SOCK_BUF); + } else if (size + http->reqofs >= HTTP_REQBUF_SZ && http->out.offset == 0) { clientProcessMiss(http); } else { debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); + http->reqofs += size; + assert(http->reqofs <= HTTP_REQBUF_SZ); storeClientCopy(http->sc, e, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ, + http->reqbuf + http->reqofs, clientCacheHit, http); } @@ -1471,7 +1325,6 @@ clientCacheHit(void *data, char *buf, ssize_t size) /* This is not the correct entity for this request. We need * to requery the cache. */ - memFree(buf, MEM_CLIENT_SOCK_BUF); http->entry = NULL; storeUnregister(http->sc, e, http); http->sc = NULL; @@ -1485,12 +1338,10 @@ clientCacheHit(void *data, char *buf, ssize_t size) case VARY_CANCEL: /* varyEvaluateMatch found a object loop. Process as miss */ debug(33, 1) ("clientProcessHit: Vary object loop!\n"); - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); return; } if (r->method == METHOD_PURGE) { - memFree(buf, MEM_CLIENT_SOCK_BUF); http->entry = NULL; storeUnregister(http->sc, e, http); http->sc = NULL; @@ -1552,7 +1403,6 @@ clientCacheHit(void *data, char *buf, ssize_t size) http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } - memFree(buf, MEM_CLIENT_SOCK_BUF); } else if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client @@ -1560,7 +1410,6 @@ clientCacheHit(void *data, char *buf, ssize_t size) if (mem->reply->sline.status != HTTP_OK) { debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); - memFree(buf, MEM_CLIENT_SOCK_BUF); http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } else if (modifiedSince(e, http->request)) { @@ -1570,7 +1419,6 @@ clientCacheHit(void *data, char *buf, ssize_t size) time_t timestamp = e->timestamp; MemBuf mb = httpPacked304Reply(e->mem_obj->reply); http->log_type = LOG_TCP_IMS_HIT; - memFree(buf, MEM_CLIENT_SOCK_BUF); storeUnregister(http->sc, e, http); http->sc = NULL; storeUnlockObject(e); @@ -1598,168 +1446,6 @@ clientCacheHit(void *data, char *buf, ssize_t size) } } -/* put terminating boundary for multiparts */ -static void -clientPackTermBound(String boundary, MemBuf * mb) -{ - memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(boundary)); - debug(33, 6) ("clientPackTermBound: buf offset: %ld\n", (long int) mb->size); -} - -/* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */ -static void -clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb) -{ - HttpHeader hdr; - Packer p; - assert(rep); - assert(spec); - - /* put boundary */ - debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary)); - /* rfc2046 requires to _prepend_ boundary with ! */ - memBufPrintf(mb, "\r\n--%s\r\n", strBuf(boundary)); - - /* stuff the header with required entries and pack it */ - httpHeaderInit(&hdr, hoReply); - if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) - httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); - httpHeaderAddContRange(&hdr, *spec, rep->content_length); - packerToMemInit(&p, mb); - httpHeaderPackInto(&hdr, &p); - packerClean(&p); - httpHeaderClean(&hdr); - - /* append (we packed a header, not a reply) */ - memBufPrintf(mb, crlf); -} - -/* - * extracts a "range" from *buf and appends them to mb, updating - * all offsets and such. - */ -static void -clientPackRange(clientHttpRequest * http, - HttpHdrRangeIter * i, - const char **buf, - ssize_t * size, - MemBuf * mb) -{ - const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; - off_t body_off = http->out.offset - i->prefix_size; - assert(*size > 0); - assert(i->spec); - /* - * intersection of "have" and "need" ranges must not be empty - */ - assert(body_off < i->spec->offset + i->spec->length); - assert(body_off + *size > i->spec->offset); - /* - * put boundary and headers at the beginning of a range in a - * multi-range - */ - if (http->request->range->specs.count > 1 && i->debt_size == i->spec->length) { - assert(http->entry->mem_obj); - clientPackRangeHdr( - http->entry->mem_obj->reply, /* original reply */ - i->spec, /* current range */ - i->boundary, /* boundary, the same for all */ - mb - ); - } - /* - * append content - */ - debug(33, 3) ("clientPackRange: appending %ld bytes\n", (long int) copy_sz); - memBufAppend(mb, *buf, copy_sz); - /* - * update offsets - */ - *size -= copy_sz; - i->debt_size -= copy_sz; - body_off += copy_sz; - *buf += copy_sz; - http->out.offset = body_off + i->prefix_size; /* sync */ - /* - * paranoid check - */ - assert(*size >= 0 && i->debt_size >= 0); -} - -/* returns true if there is still data available to pack more ranges - * increments iterator "i" - * used by clientPackMoreRanges */ -static int -clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size) -{ - /* first update "i" if needed */ - if (!i->debt_size) { - if ((i->spec = httpHdrRangeGetSpec(http->request->range, &i->pos))) - i->debt_size = i->spec->length; - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - /* continue condition: need_more_data && have_more_data */ - return i->spec && size > 0; -} - -/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ -/* returns true if we need more data */ -static int -clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb) -{ - HttpHdrRangeIter *i = &http->range_iter; - /* offset in range specs does not count the prefix of an http msg */ - off_t body_off = http->out.offset - i->prefix_size; - assert(size >= 0); - /* check: reply was parsed and range iterator was initialized */ - assert(i->prefix_size > 0); - /* filter out data according to range specs */ - while (clientCanPackMoreRanges(http, i, size)) { - off_t start; /* offset of still missing data */ - assert(i->spec); - start = i->spec->offset + i->spec->length - i->debt_size; - debug(33, 3) ("clientPackMoreRanges: in: offset: %ld size: %ld\n", - (long int) body_off, (long int) size); - debug(33, 3) ("clientPackMoreRanges: out: start: %ld spec[%ld]: [%ld, %ld), len: %ld debt: %ld\n", - (long int) start, (long int) i->pos, (long int) i->spec->offset, (long int) (i->spec->offset + i->spec->length), (long int) i->spec->length, (long int) i->debt_size); - assert(body_off <= start); /* we did not miss it */ - /* skip up to start */ - if (body_off + size > start) { - const size_t skip_size = start - body_off; - body_off = start; - size -= skip_size; - buf += skip_size; - } else { - /* has not reached start yet */ - body_off += size; - size = 0; - buf = NULL; - } - /* put next chunk if any */ - if (size) { - http->out.offset = body_off + i->prefix_size; /* sync */ - clientPackRange(http, i, &buf, &size, mb); - body_off = http->out.offset - i->prefix_size; /* sync */ - } - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %ld size: %ld need_more: %ld\n", - (long int) body_off, (long int) size, (long int) i->debt_size); - if (i->debt_size) { - debug(33, 3) ("clientPackMoreRanges: need more: spec[%ld]: [%ld, %ld), len: %ld\n", - (long int) i->pos, (long int) i->spec->offset, (long int) (i->spec->offset + i->spec->length), (long int) i->spec->length); - /* skip the data we do not need if possible */ - if (i->debt_size == i->spec->length) /* at the start of the cur. spec */ - body_off = i->spec->offset; - else - assert(body_off == i->spec->offset + i->spec->length - i->debt_size); - } else if (http->request->range->specs.count > 1) { - /* put terminating boundary for multiparts */ - clientPackTermBound(i->boundary, mb); - } - http->out.offset = body_off + i->prefix_size; /* sync */ - return i->debt_size > 0; -} static int clientReplyBodyTooLarge(HttpReply * rep, ssize_t clen) @@ -1830,31 +1516,29 @@ clientSendMoreData(void *data, char *buf, ssize_t size) MemBuf mb; ssize_t check_size = 0; debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http->uri, (int) size); - assert(size <= CLIENT_SOCK_SZ); + assert(size <= HTTP_REQBUF_SZ); assert(http->request != NULL); dlinkDelete(&http->active, &ClientActiveRequests); dlinkAdd(http, &http->active, &ClientActiveRequests); debug(33, 5) ("clientSendMoreData: FD %d '%s', out.offset=%ld \n", fd, storeUrl(entry), (long int) http->out.offset); + /* update size of the request */ + http->reqsize = size + http->reqofs; if (conn->chr != http) { /* there is another object in progress, defer this one */ debug(33, 2) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size < 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size == 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (http->out.offset == 0) { @@ -1886,7 +1570,6 @@ clientSendMoreData(void *data, char *buf, ssize_t size) body_size = size - rep->hdr_sz; assert(body_size >= 0); body_buf = buf + rep->hdr_sz; - http->range_iter.prefix_size = rep->hdr_sz; debug(33, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n", (int) body_size, rep->hdr_sz); ch = aclChecklistCreate(Config.accessList.reply, http->request, NULL); @@ -1914,23 +1597,22 @@ clientSendMoreData(void *data, char *buf, ssize_t size) return; } aclChecklistFree(ch); - } else if (size < CLIENT_SOCK_SZ && entry->store_status == STORE_PENDING) { + } else if (size + http->reqofs < HTTP_REQBUF_SZ && entry->store_status == STORE_PENDING) { /* wait for more to arrive */ + http->reqofs += size; + assert(http->reqofs <= HTTP_REQBUF_SZ); storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->out.offset + http->reqofs, + HTTP_REQBUF_SZ - http->reqofs, + http->reqbuf + http->reqofs, clientSendMoreData, http); return; } - /* reset range iterator */ - http->range_iter.pos = HttpHdrRangeInitPos; - } else if (!http->request->range) { - /* Avoid copying to MemBuf for non-range requests */ - /* Note, if we're here, then 'rep' is known to be NULL */ + } else { + /* Avoid copying to MemBuf if we know "rep" is NULL, and we only have a body */ http->out.offset += body_size; + assert(rep == NULL); comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); /* NULL because clientWriteBodyComplete frees it */ return; @@ -1967,25 +1649,14 @@ clientSendMoreData(void *data, char *buf, ssize_t size) } else { memBufDefInit(&mb); } - /* append body if any */ - if (http->request->range) { - /* Only GET requests should have ranges */ - assert(http->request->method == METHOD_GET); - /* clientPackMoreRanges() updates http->out.offset */ - /* force the end of the transfer if we are done */ - if (!clientPackMoreRanges(http, body_buf, body_size, &mb)) - http->flags.done_copying = 1; - } else if (body_buf && body_size) { + if (body_buf && body_size) { http->out.offset += body_size; check_size += body_size; memBufAppend(&mb, body_buf, body_size); } - if (!http->request->range && http->request->method == METHOD_GET) - assert(check_size == size); /* write */ comm_write_mbuf(fd, mb, clientWriteComplete, http); /* if we don't do it, who will? */ - memFree(buf, MEM_CLIENT_SOCK_BUF); } /* @@ -2002,7 +1673,6 @@ clientWriteBodyComplete(int fd, char *buf, size_t size, int errflag, void *data) * (second) argument, so we pass in NULL. */ clientWriteComplete(fd, NULL, size, errflag, data); - memFree(buf, MEM_CLIENT_SOCK_BUF); } static void @@ -2047,11 +1717,11 @@ clientKeepaliveNextRequest(clientHttpRequest * http) if (0 == storeClientCopyPending(http->sc, entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); + http->reqofs = 0; storeClientCopy(http->sc, entry, http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + HTTP_REQBUF_SZ, + http->reqbuf, clientSendMoreData, http); } @@ -2106,11 +1776,11 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da * storage manager. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n"); + http->reqofs = 0; storeClientCopy(http->sc, entry, http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + HTTP_REQBUF_SZ, + http->reqbuf, clientSendMoreData, http); } @@ -2143,42 +1813,6 @@ clientProcessOnlyIfCachedMiss(clientHttpRequest * http) errorAppendEntry(http->entry, err); } -/* - * Return true if we should force a cache miss on this range request. - * entry must be non-NULL. - */ -static int -clientCheckRangeForceMiss(StoreEntry * entry, HttpHdrRange * range) -{ - /* - * If the range_offset_limit is NOT in effect, there - * is no reason to force a miss. - */ - if (0 == httpHdrRangeOffsetLimit(range)) - return 0; - /* - * Here, we know it's possibly a hit. If we already have the - * whole object cached, we won't force a miss. - */ - if (STORE_OK == entry->store_status) - return 0; /* we have the whole object */ - /* - * Now we have a hit on a PENDING object. We need to see - * if the part we want is already cached. If so, we don't - * force a miss. - */ - assert(NULL != entry->mem_obj); - if (httpHdrRangeFirstOffset(range) <= entry->mem_obj->inmem_hi) - return 0; - /* - * Even though we have a PENDING copy of the object, we - * don't want to wait to reach the first range offset, - * so we force a miss for a new range request to the - * origin. - */ - return 1; -} - static log_type clientProcessRequest2(clientHttpRequest * http) { @@ -2239,22 +1873,10 @@ clientProcessRequest2(clientHttpRequest * http) http->entry = NULL; return LOG_TCP_CLIENT_REFRESH_MISS; } - if (NULL == r->range) { - (void) 0; - } else if (httpHdrRangeWillBeComplex(r->range)) { - /* - * Some clients break if we return "200 OK" for a Range - * request. We would have to return "200 OK" for a _complex_ - * Range request that is also a HIT. Thus, let's prevent HITs - * on complex Range requests - */ - debug(33, 3) ("clientProcessRequest2: complex range MISS\n"); - http->entry = NULL; - return LOG_TCP_MISS; - } else if (clientCheckRangeForceMiss(e, r->range)) { - debug(33, 3) ("clientProcessRequest2: forcing miss due to range_offset_limit\n"); - http->entry = NULL; - return LOG_TCP_MISS; + /* We don't cache any range requests (for now!) -- adrian */ + if (r->flags.range) { + http->entry = NULL; + return LOG_TCP_MISS; } debug(33, 3) ("clientProcessRequest2: default HIT\n"); http->entry = e; @@ -2311,11 +1933,12 @@ clientProcessRequest(clientHttpRequest * http) #if DELAY_POOLS delaySetStoreClient(http->sc, delayClient(r)); #endif + assert(http->log_type == LOG_TCP_HIT); + http->reqofs = 0; storeClientCopy(http->sc, http->entry, http->out.offset, - http->out.offset, - CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + HTTP_REQBUF_SZ, + http->reqbuf, clientCacheHit, http); } else { @@ -2399,7 +2022,7 @@ parseHttpRequestAbort(ConnStateData * conn, const char *uri) http->req_sz = conn->in.offset; http->uri = xstrdup(uri); http->log_uri = xstrndup(uri, MAX_URL); - http->range_iter.boundary = StringNull; + http->reqbuf = http->norm_reqbuf; dlinkAdd(http, &http->active, &ClientActiveRequests); return http; } @@ -2542,7 +2165,7 @@ parseHttpRequest(ConnStateData * conn, method_t * method_p, int *status, http->conn = conn; http->start = current_time; http->req_sz = prefix_sz; - http->range_iter.boundary = StringNull; + http->reqbuf = http->norm_reqbuf; *prefix_p = xmalloc(prefix_sz + 1); xmemcpy(*prefix_p, conn->in.buf, prefix_sz); *(*prefix_p + prefix_sz) = '\0'; @@ -2660,7 +2283,7 @@ parseHttpRequest(ConnStateData * conn, method_t * method_p, int *status, vport, url); } else { if (vport_mode) - vport = natLookup.nl_realport; + vport = ntohs(natLookup.nl_realport); snprintf(http->uri, url_sz, "http://%s:%d%s", inet_ntoa(natLookup.nl_realip), vport, url); diff --git a/src/defines.h b/src/defines.h index e580769f54..8eadc43a36 100644 --- a/src/defines.h +++ b/src/defines.h @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.98 2002/02/18 23:40:35 hno Exp $ + * $Id: defines.h,v 1.99 2002/02/26 15:48:14 adrian Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -146,6 +146,7 @@ #define LOG_DISABLE 0 #define SM_PAGE_SIZE 4096 +#define MAX_CLIENT_BUF_SZ 4096 #define EBIT_SET(flag, bit) ((void)((flag) |= ((1L<<(bit))))) #define EBIT_CLR(flag, bit) ((void)((flag) &= ~((1L<<(bit))))) @@ -269,8 +270,6 @@ */ #define PEER_TCP_MAGIC_COUNT 10 -#define CLIENT_SOCK_SZ 4096 - #define URI_WHITESPACE_STRIP 0 #define URI_WHITESPACE_ALLOW 1 #define URI_WHITESPACE_ENCODE 2 @@ -296,4 +295,5 @@ #define O_BINARY 0 #endif +#define HTTP_REQBUF_SZ 4096 #endif /* SQUID_DEFINES_H */ diff --git a/src/enums.h b/src/enums.h index 0cab6b20a5..1bf29fafe5 100644 --- a/src/enums.h +++ b/src/enums.h @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.204 2002/02/13 19:34:01 hno Exp $ + * $Id: enums.h,v 1.205 2002/02/26 15:48:14 adrian Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -574,7 +574,6 @@ typedef enum { MEM_CACHE_DIGEST, #endif MEM_CLIENT_INFO, - MEM_CLIENT_SOCK_BUF, MEM_LINK_LIST, MEM_DLINK_NODE, MEM_DONTFREE, @@ -721,6 +720,18 @@ enum { VARY_CANCEL }; +/* + * Store digest state enum + */ +typedef enum { + DIGEST_READ_NONE, + DIGEST_READ_REPLY, + DIGEST_READ_HEADERS, + DIGEST_READ_CBLOCK, + DIGEST_READ_MASK, + DIGEST_READ_DONE +} digest_read_state_t; + /* CygWin & Windows NT Port */ #if defined(_SQUID_MSWIN_) || defined(_SQUID_CYGWIN_) /* diff --git a/src/fs/Makefile.in b/src/fs/Makefile.in index f700805446..cb56568ff5 100644 --- a/src/fs/Makefile.in +++ b/src/fs/Makefile.in @@ -15,7 +15,7 @@ # Makefile for storage modules in the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/11/29 11:17:32 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:29 adrian Exp $ # SHELL = @SHELL@ diff --git a/src/fs/diskd/Makefile.in b/src/fs/diskd/Makefile.in index 261fa85e6d..13d4dc0528 100644 --- a/src/fs/diskd/Makefile.in +++ b/src/fs/diskd/Makefile.in @@ -16,7 +16,7 @@ # # Makefile for the DISKD storage driver for the Squid Object Cache server # -# $Id: Makefile.in,v 1.6 2001/11/29 11:17:35 hno Exp $ +# $Id: Makefile.in,v 1.7 2002/02/26 15:48:29 adrian Exp $ # SHELL = @SHELL@ diff --git a/src/http.cc b/src/http.cc index 7a8ffcb25e..abd1dffa05 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.386 2002/01/28 17:51:27 hno Exp $ + * $Id: http.cc,v 1.387 2002/02/26 15:48:14 adrian Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -729,7 +729,6 @@ httpBuildRequestHeader(request_t * request, LOCAL_ARRAY(char, bbuf, BBUF_SZ); String strConnection = StringNull; const HttpHeader *hdr_in = &orig_request->header; - int we_do_ranges; const HttpHeaderEntry *e; String strVia; String strFwd; @@ -739,27 +738,6 @@ httpBuildRequestHeader(request_t * request, if (request->lastmod > -1 && request->method == METHOD_GET) httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); - /* decide if we want to do Ranges ourselves - * (and fetch the whole object now) - * We want to handle Ranges ourselves iff - * - we can actually parse client Range specs - * - the specs are expected to be simple enough (e.g. no out-of-order ranges) - * - reply will be cachable - * (If the reply will be uncachable we have to throw it away after - * serving this request, so it is better to forward ranges to - * the server and fetch only the requested content) - */ - if (NULL == orig_request->range) - we_do_ranges = 0; - else if (!orig_request->flags.cachable) - we_do_ranges = 0; - else if (httpHdrRangeOffsetLimit(orig_request->range)) - we_do_ranges = 0; - else - we_do_ranges = 1; - debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", - orig_request->range, orig_request->flags.cachable, we_do_ranges); - strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); while ((e = httpHeaderGetEntry(hdr_in, &pos))) { debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", @@ -819,12 +797,6 @@ httpBuildRequestHeader(request_t * request, httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); } break; - case HDR_RANGE: - case HDR_IF_RANGE: - case HDR_REQUEST_RANGE: - if (!we_do_ranges) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; case HDR_PROXY_CONNECTION: case HDR_CONNECTION: case HDR_VIA: diff --git a/src/mem.cc b/src/mem.cc index f2da9c69de..8bc8ff6d62 100644 --- a/src/mem.cc +++ b/src/mem.cc @@ -1,6 +1,6 @@ /* - * $Id: mem.cc,v 1.61 2002/02/13 19:34:02 hno Exp $ + * $Id: mem.cc,v 1.62 2002/02/26 15:48:15 adrian Exp $ * * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived @@ -284,7 +284,6 @@ memInit(void) memDataInit(MEM_16K_BUF, "16K Buffer", 16384, 10); memDataInit(MEM_32K_BUF, "32K Buffer", 32768, 10); memDataInit(MEM_64K_BUF, "64K Buffer", 65536, 10); - memDataInit(MEM_CLIENT_SOCK_BUF, "Client Socket Buffer", CLIENT_SOCK_SZ, 0); memDataInit(MEM_ACL, "acl", sizeof(acl), 0); memDataInit(MEM_ACL_DENY_INFO_LIST, "acl_deny_info_list", sizeof(acl_deny_info_list), 0); diff --git a/src/net_db.cc b/src/net_db.cc index 554ec4833c..db9f838240 100644 --- a/src/net_db.cc +++ b/src/net_db.cc @@ -1,6 +1,6 @@ /* - * $Id: net_db.cc,v 1.158 2001/06/12 23:52:45 wessels Exp $ + * $Id: net_db.cc,v 1.159 2002/02/26 15:48:15 adrian Exp $ * * DEBUG: section 38 Network Measurement Database * AUTHOR: Duane Wessels @@ -33,19 +33,35 @@ * */ +/* + * XXX XXX XXX + * + * This code may be slightly broken now. If you're getting consistent + * (sometimes working) corrupt data exchanges, please contact adrian + * (adrian@squid-cache.org) to sort them out. + */ + #include "squid.h" #if USE_ICMP +#define NETDB_REQBUF_SZ 4096 + +typedef enum { + STATE_NONE, + STATE_HEADER, + STATE_BODY +} netdb_conn_state_t; typedef struct { peer *p; StoreEntry *e; store_client *sc; request_t *r; - off_t seen; off_t used; size_t buf_sz; - char *buf; + char buf[NETDB_REQBUF_SZ]; + int buf_ofs; + netdb_conn_state_t connstate; } netdbExchangeState; static hash_table *addr_table = NULL; @@ -526,8 +542,9 @@ netdbFreeNameEntry(void *data) memFree(x, MEM_NET_DB_NAME); } + static void -netdbExchangeHandleReply(void *data, char *buf, ssize_t size) +netdbExchangeHandleReply(void *data, char *notused, ssize_t retsize) { netdbExchangeState *ex = data; int rec_sz = 0; @@ -540,40 +557,68 @@ netdbExchangeHandleReply(void *data, char *buf, ssize_t size) HttpReply *rep; size_t hdr_sz; int nused = 0; + int size; + int oldbufofs = ex->buf_ofs; + rec_sz = 0; rec_sz += 1 + sizeof(addr.s_addr); rec_sz += 1 + sizeof(int); rec_sz += 1 + sizeof(int); - ex->seen = ex->used + size; - debug(38, 3) ("netdbExchangeHandleReply: %d bytes\n", (int) size); + debug(38, 3) ("netdbExchangeHandleReply: %d read bytes\n", (int) retsize); if (!cbdataValid(ex->p)) { debug(38, 3) ("netdbExchangeHandleReply: Peer became invalid\n"); netdbExchangeDone(ex); return; } debug(38, 3) ("netdbExchangeHandleReply: for '%s:%d'\n", ex->p->host, ex->p->http_port); - p = buf; - if (0 == ex->used) { + p = ex->buf; + + /* Get the size of the buffer now */ + size = ex->buf_ofs + retsize; + debug(38, 3) ("netdbExchangeHandleReply: %d bytes buf\n", (int) size); + + /* Check if we're still doing headers */ + if (ex->connstate == STATE_HEADER) { + + ex->buf_ofs += retsize; + /* skip reply headers */ - if ((hdr_sz = headersEnd(p, size))) { + if ((hdr_sz = headersEnd(p, ex->buf_ofs))) { debug(38, 5) ("netdbExchangeHandleReply: hdr_sz = %d\n", hdr_sz); rep = ex->e->mem_obj->reply; if (0 == rep->sline.status) - httpReplyParse(rep, buf, hdr_sz); + httpReplyParse(rep, ex->buf, hdr_sz); debug(38, 3) ("netdbExchangeHandleReply: reply status %d\n", rep->sline.status); if (HTTP_OK != rep->sline.status) { netdbExchangeDone(ex); return; } - assert(size >= hdr_sz); - ex->used += hdr_sz; - size -= hdr_sz; + assert(ex->buf_ofs >= hdr_sz); + + /* + * Now, point p to the part of the buffer where the data + * starts, and update the size accordingly + */ + assert(ex->used == 0); + ex->used = hdr_sz; + size = ex->buf_ofs - hdr_sz; p += hdr_sz; + + /* Finally, set the conn state mode to STATE_BODY */ + ex->connstate = STATE_BODY; } else { - size = 0; + /* Have more headers .. */ + storeClientCopy(ex->sc, ex->e, ex->buf_ofs, + ex->buf_sz - ex->buf_ofs, ex->buf + ex->buf_ofs, + netdbExchangeHandleReply, ex); + return; } } + + assert(ex->connstate == STATE_BODY); + + /* If we get here, we have some body to parse .. */ debug(38, 5) ("netdbExchangeHandleReply: start parsing loop, size = %d\n", size); while (size >= rec_sz) { @@ -612,27 +657,51 @@ netdbExchangeHandleReply(void *data, char *buf, ssize_t size) ex->used += rec_sz; size -= rec_sz; p += rec_sz; - /* - * This is a fairly cpu-intensive loop, break after adding - * just a few - */ - if (++nused == 20) - break; + nused++; } + + /* + * Copy anything that is left over to the beginning of the buffer, + * and adjust buf_ofs accordingly + */ + + /* + * Evilly, size refers to the buf size left now, + * ex->buf_ofs is the original buffer size, so just copy that + * much data over + */ + memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size); + ex->buf_ofs = size; + + /* + * And don't re-copy the remaining data .. + */ + ex->used += size; + + /* + * Now the tricky bit - size _included_ the leftover bit from the _last_ + * storeClientCopy. We don't want to include that, or our offset will be wrong. + * So, don't count the size of the leftover buffer we began with. + * This can _disappear_ when we're not tracking offsets .. + */ + ex->used -= oldbufofs; + + debug(38, 3) ("netdbExchangeHandleReply: size left over in this buffer: %d bytes\n", size); + debug(38, 3) ("netdbExchangeHandleReply: used %d entries, (x %d bytes) == %d bytes total\n", nused, rec_sz, nused * rec_sz); - debug(38, 3) ("netdbExchangeHandleReply: seen %d, used %d\n", ex->seen, ex->used); + debug(38, 3) ("netdbExchangeHandleReply: used %ld\n",(long int) ex->used); if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) { debug(38, 3) ("netdbExchangeHandleReply: ENTRY_ABORTED\n"); netdbExchangeDone(ex); } else if (ex->e->store_status == STORE_PENDING) { debug(38, 3) ("netdbExchangeHandleReply: STORE_PENDING\n"); - storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, - ex->buf, netdbExchangeHandleReply, ex); - } else if (ex->seen < ex->e->mem_obj->inmem_hi) { + storeClientCopy(ex->sc, ex->e, ex->used, ex->buf_sz - ex->buf_ofs, + ex->buf + ex->buf_ofs, netdbExchangeHandleReply, ex); + } else if (ex->used < ex->e->mem_obj->inmem_hi) { debug(38, 3) ("netdbExchangeHandleReply: ex->e->mem_obj->inmem_hi\n"); - storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, - ex->buf, netdbExchangeHandleReply, ex); + storeClientCopy(ex->sc, ex->e, ex->used, ex->buf_sz - ex->buf_ofs, + ex->buf + ex->buf_ofs, netdbExchangeHandleReply, ex); } else { debug(38, 3) ("netdbExchangeHandleReply: Done\n"); netdbExchangeDone(ex); @@ -644,7 +713,6 @@ netdbExchangeDone(void *data) { netdbExchangeState *ex = data; debug(38, 3) ("netdbExchangeDone: %s\n", storeUrl(ex->e)); - memFree(ex->buf, MEM_4K_BUF); requestUnlink(ex->r); storeUnregister(ex->sc, ex->e, ex); storeUnlockObject(ex->e); @@ -1007,13 +1075,13 @@ netdbExchangeStart(void *data) requestLink(ex->r); assert(NULL != ex->r); httpBuildVersion(&ex->r->http_ver, 1, 0); + ex->connstate = STATE_HEADER; ex->e = storeCreateEntry(uri, uri, null_request_flags, METHOD_GET); - ex->buf_sz = 4096; - ex->buf = memAllocate(MEM_4K_BUF); + ex->buf_sz = NETDB_REQBUF_SZ; assert(NULL != ex->e); ex->sc = storeClientListAdd(ex->e, ex); - storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, - ex->buf, netdbExchangeHandleReply, ex); + storeClientCopy(ex->sc, ex->e, 0, ex->buf_sz, ex->buf, + netdbExchangeHandleReply, ex); ex->r->flags.loopdetect = 1; /* cheat! -- force direct */ if (p->login) xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ); diff --git a/src/peer_digest.cc b/src/peer_digest.cc index 305b5b6bed..fa0385cf80 100644 --- a/src/peer_digest.cc +++ b/src/peer_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: peer_digest.cc,v 1.83 2001/10/24 06:55:44 hno Exp $ + * $Id: peer_digest.cc,v 1.84 2002/02/26 15:48:15 adrian Exp $ * * DEBUG: section 72 Peer Digest Routines * AUTHOR: Alex Rousskov @@ -46,10 +46,11 @@ static void peerDigestSetCheck(PeerDigest * pd, time_t delay); static void peerDigestClean(PeerDigest *); static EVH peerDigestCheck; static void peerDigestRequest(PeerDigest * pd); -static STCB peerDigestFetchReply; -static STCB peerDigestSwapInHeaders; -static STCB peerDigestSwapInCBlock; -static STCB peerDigestSwapInMask; +static STCB peerDigestHandleReply; +static int peerDigestFetchReply(void *, char *, int); +static int peerDigestSwapInHeaders(void *, char *, int); +static int peerDigestSwapInCBlock(void *, char *, int); +static int peerDigestSwapInMask(void *, char *, int); static int peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name); static void peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason); static void peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason); @@ -302,6 +303,7 @@ peerDigestRequest(PeerDigest * pd) fetch->request = requestLink(req); fetch->pd = pd; fetch->offset = 0; + fetch->state = DIGEST_READ_REPLY; /* update timestamps */ fetch->start_time = squid_curtime; @@ -330,12 +332,105 @@ peerDigestRequest(PeerDigest * pd) fwdStart(-1, e, req); cbdataLock(fetch); cbdataLock(fetch->pd); - storeClientCopy(fetch->sc, e, 0, 0, 4096, memAllocate(MEM_4K_BUF), - peerDigestFetchReply, fetch); + storeClientCopy(fetch->sc, e, 0, SM_PAGE_SIZE, fetch->buf, + peerDigestHandleReply, fetch); } -/* wait for full http headers to be received then parse them */ + +/* Handle the data copying .. */ + +/* + * This routine handles the copy data and then redirects the + * copy to a bunch of subfunctions depending upon the copy state. + * It also tracks the buffer offset and "seen", since I'm actually + * not interested in rewriting everything to suit my little idea. + */ static void +peerDigestHandleReply(void *data, char *buf, ssize_t copysize) +{ + DigestFetchState *fetch = data; + PeerDigest *pd = fetch->pd; + int retsize = -1; + digest_read_state_t prevstate; + int newsize; + + assert(pd && buf); + + /* Update the buffer size */ + fetch->bufofs += copysize; + + assert(fetch->bufofs <= SM_PAGE_SIZE); + + /* If we've fetched enough, return */ + if (peerDigestFetchedEnough(fetch, fetch->buf, fetch->bufofs, "peerDigestHandleReply")) + return; + + /* Call the right function based on the state */ + /* (Those functions will update the state if needed) */ + cbdataLock(fetch); + + /* Repeat this loop until we're out of data OR the state changes */ + /* (So keep going if the state has changed and we still have data */ + do { + prevstate = fetch->state; + switch(fetch->state) { + case DIGEST_READ_REPLY: + retsize = peerDigestFetchReply(data, fetch->buf, fetch->bufofs); + break; + case DIGEST_READ_HEADERS: + retsize = peerDigestSwapInHeaders(data, fetch->buf, fetch->bufofs); + break; + case DIGEST_READ_CBLOCK: + retsize = peerDigestSwapInCBlock(data, fetch->buf, fetch->bufofs); + break; + case DIGEST_READ_MASK: + retsize = peerDigestSwapInMask(data, fetch->buf, fetch->bufofs); + break; + case DIGEST_READ_NONE: + break; + case DIGEST_READ_DONE: + goto finish; + break; + default: + fatal("Bad digest transfer mode!\n"); + } + + if (retsize < 0) + goto finish; + /* + * The returned size indicates how much of the buffer was read - + * so move the remainder of the buffer to the beginning + * and update the bufofs / bufsize + */ + newsize = fetch->bufofs - retsize; + xmemmove(fetch->buf, fetch->buf + retsize, fetch->bufofs - newsize); + fetch->bufofs = newsize; + + } while (prevstate != fetch->state && fetch->bufofs > 0); + + /* Update the copy offset */ + fetch->offset += copysize; + + /* Schedule another copy */ + if (cbdataValid(fetch)) { + storeClientCopy(fetch->sc, fetch->entry, fetch->offset, SM_PAGE_SIZE - fetch->bufofs, + fetch->buf + fetch->bufofs, peerDigestHandleReply, fetch); + } +finish: + /* Unlock our data - we've finished with it for now */ + cbdataUnlock(fetch); +} + + + +/* wait for full http headers to be received then parse them */ +/* + * This routine handles parsing the reply line. + * If the reply line indicates an OK, the same data is thrown + * to SwapInHeaders(). If the reply line is a NOT_MODIFIED, + * we simply stop parsing. + */ +static int peerDigestFetchReply(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; @@ -344,8 +439,9 @@ peerDigestFetchReply(void *data, char *buf, ssize_t size) assert(pd && buf); assert(!fetch->offset); + assert(fetch->state == DIGEST_READ_REPLY); if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestFetchReply")) - return; + return -1; if ((hdr_size = headersEnd(buf, size))) { http_status status; @@ -388,34 +484,37 @@ peerDigestFetchReply(void *data, char *buf, ssize_t size) } else { /* some kind of a bug */ peerDigestFetchAbort(fetch, buf, httpStatusLineReason(&reply->sline)); - return; + return -1; /* XXX -1 will abort stuff in ReadReply! */ } /* must have a ready-to-use store entry if we got here */ /* can we stay with the old in-memory digest? */ - if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) + if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) { peerDigestFetchStop(fetch, buf, "Not modified"); - else - storeClientCopy(fetch->sc, fetch->entry, /* have to swap in */ - 0, 0, SM_PAGE_SIZE, buf, peerDigestSwapInHeaders, fetch); + fetch->state = DIGEST_READ_DONE; + } else { + fetch->state = DIGEST_READ_HEADERS; + } } else { /* need more data, do we have space? */ if (size >= SM_PAGE_SIZE) peerDigestFetchAbort(fetch, buf, "reply header too big"); - else - storeClientCopy(fetch->sc, fetch->entry, size, 0, SM_PAGE_SIZE, buf, - peerDigestFetchReply, fetch); } + + /* We don't want to actually ack that we've handled anything, + * otherwise SwapInHeaders() won't get the reply line .. */ + return 0; } /* fetch headers from disk, pass on to SwapInCBlock */ -static void +static int peerDigestSwapInHeaders(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; size_t hdr_size; + assert(fetch->state == DIGEST_READ_HEADERS); if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInHeaders")) - return; + return -1; assert(!fetch->offset); if ((hdr_size = headersEnd(buf, size))) { @@ -426,92 +525,90 @@ peerDigestSwapInHeaders(void *data, char *buf, ssize_t size) debug(72, 1) ("peerDigestSwapInHeaders: %s status %d got cached!\n", strBuf(fetch->pd->host), fetch->entry->mem_obj->reply->sline.status); peerDigestFetchAbort(fetch, buf, "internal status error"); - return; + return -1; } - fetch->offset += hdr_size; - storeClientCopy(fetch->sc, fetch->entry, size, fetch->offset, - SM_PAGE_SIZE, buf, - peerDigestSwapInCBlock, fetch); + fetch->state = DIGEST_READ_CBLOCK; + return hdr_size; /* Say how much data we read */ } else { /* need more data, do we have space? */ - if (size >= SM_PAGE_SIZE) + if (size >= SM_PAGE_SIZE) { peerDigestFetchAbort(fetch, buf, "stored header too big"); - else - storeClientCopy(fetch->sc, fetch->entry, size, 0, SM_PAGE_SIZE, buf, - peerDigestSwapInHeaders, fetch); + return -1; + } else { + return 0; /* We need to read more to parse .. */ + } } + fatal("peerDigestSwapInHeaders() - shouldn't get here!\n"); } -static void +static int peerDigestSwapInCBlock(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; + assert(fetch->state == DIGEST_READ_CBLOCK); if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInCBlock")) - return; + return -1; if (size >= StoreDigestCBlockSize) { PeerDigest *pd = fetch->pd; HttpReply *rep = fetch->entry->mem_obj->reply; - const int seen = fetch->offset + size; assert(pd && rep); if (peerDigestSetCBlock(pd, buf)) { /* XXX: soon we will have variable header size */ - fetch->offset += StoreDigestCBlockSize; /* switch to CD buffer and fetch digest guts */ - memFree(buf, MEM_4K_BUF); buf = NULL; assert(pd->cd->mask); - storeClientCopy(fetch->sc, fetch->entry, - seen, - fetch->offset, - pd->cd->mask_size, - pd->cd->mask, - peerDigestSwapInMask, fetch); + fetch->state = DIGEST_READ_MASK; + return StoreDigestCBlockSize; } else { peerDigestFetchAbort(fetch, buf, "invalid digest cblock"); + return -1; } } else { /* need more data, do we have space? */ - if (size >= SM_PAGE_SIZE) + if (size >= SM_PAGE_SIZE) { peerDigestFetchAbort(fetch, buf, "digest cblock too big"); - else - storeClientCopy(fetch->sc, fetch->entry, size, 0, SM_PAGE_SIZE, buf, - peerDigestSwapInCBlock, fetch); + return -1; + } else { + return 0; /* We need more data */ + } } + fatal("peerDigestSwapInCBlock(): shouldn't get here!\n"); } -static void +static int peerDigestSwapInMask(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; PeerDigest *pd; - /* NOTE! buf points to the middle of pd->cd->mask! */ - if (peerDigestFetchedEnough(fetch, NULL, size, "peerDigestSwapInMask")) - return; - pd = fetch->pd; assert(pd->cd && pd->cd->mask); - fetch->offset += size; + /* + * NOTENOTENOTENOTENOTE: buf doesn't point to pd->cd->mask anymore! + * we need to do the copy ourselves! + */ + xmemcpy(pd->cd->mask + fetch->mask_offset, buf, size); + + /* NOTE! buf points to the middle of pd->cd->mask! */ + if (peerDigestFetchedEnough(fetch, NULL, size, "peerDigestSwapInMask")) + return -1; + fetch->mask_offset += size; if (fetch->mask_offset >= pd->cd->mask_size) { debug(72, 2) ("peerDigestSwapInMask: Done! Got %d, expected %d\n", fetch->mask_offset, pd->cd->mask_size); assert(fetch->mask_offset == pd->cd->mask_size); assert(peerDigestFetchedEnough(fetch, NULL, 0, "peerDigestSwapInMask")); + return -1; /* XXX! */ } else { - const size_t buf_sz = pd->cd->mask_size - fetch->mask_offset; - assert(buf_sz > 0); - storeClientCopy(fetch->sc, fetch->entry, - fetch->offset, - fetch->offset, - buf_sz, - pd->cd->mask + fetch->mask_offset, - peerDigestSwapInMask, fetch); + /* We always read everything, so return so */ + return size; } + fatal("peerDigestSwapInMask(): shouldn't get here!\n"); } static int @@ -631,8 +728,6 @@ peerDigestReqFinish(DigestFetchState * fetch, char *buf, peerDigestPDFinish(fetch, pcb_valid, err); if (fcb_valid) peerDigestFetchFinish(fetch, err); - if (buf) - memFree(buf, MEM_4K_BUF); } diff --git a/src/protos.h b/src/protos.h index af754984dc..6dfddaecc8 100644 --- a/src/protos.h +++ b/src/protos.h @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.427 2002/02/17 01:08:13 hno Exp $ + * $Id: protos.h,v 1.428 2002/02/26 15:48:15 adrian Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1047,7 +1047,8 @@ extern int storeSwapOutAble(const StoreEntry * e); extern store_client *storeClientListSearch(const MemObject * mem, void *data); #endif extern store_client *storeClientListAdd(StoreEntry * e, void *data); -extern void storeClientCopy(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); +extern void storeClientCopyOld(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); +extern void storeClientCopy(store_client *, StoreEntry * , off_t, size_t, char *, STCB *, void *); extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); extern int storeUnregister(store_client * sc, StoreEntry * e, void *data); extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); diff --git a/src/repl/Makefile.in b/src/repl/Makefile.in index 8fc6f5e6b8..4eede125c0 100644 --- a/src/repl/Makefile.in +++ b/src/repl/Makefile.in @@ -15,7 +15,7 @@ # Makefile for storage modules in the Squid Object Cache server # -# $Id: Makefile.in,v 1.8 2001/12/27 00:23:35 hno Exp $ +# $Id: Makefile.in,v 1.9 2002/02/26 15:48:31 adrian Exp $ # SHELL = @SHELL@ diff --git a/src/stat.cc b/src/stat.cc index fd135b36bf..bfd5e89bd4 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -1,6 +1,6 @@ /* - * $Id: stat.cc,v 1.352 2001/12/24 15:33:44 adrian Exp $ + * $Id: stat.cc,v 1.353 2002/02/26 15:48:15 adrian Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -288,8 +288,6 @@ statStoreEntry(StoreEntry * s, StoreEntry * e) storeAppendPrintf(s, "\tClient #%d, %p\n", i, sc->callback_data); storeAppendPrintf(s, "\t\tcopy_offset: %d\n", (int) sc->copy_offset); - storeAppendPrintf(s, "\t\tseen_offset: %d\n", - (int) sc->seen_offset); storeAppendPrintf(s, "\t\tcopy_size: %d\n", (int) sc->copy_size); storeAppendPrintf(s, "\t\tflags:"); diff --git a/src/store_client.cc b/src/store_client.cc index 1d1b342979..0130e25749 100644 --- a/src/store_client.cc +++ b/src/store_client.cc @@ -1,6 +1,6 @@ /* - * $Id: store_client.cc,v 1.105 2001/10/24 08:19:09 hno Exp $ + * $Id: store_client.cc,v 1.106 2002/02/26 15:48:15 adrian Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -136,8 +136,8 @@ storeClientListAdd(StoreEntry * e, void *data) sc = cbdataAlloc(store_client); cbdataLock(data); /* locked while we point to it */ sc->callback_data = data; - sc->seen_offset = 0; sc->copy_offset = 0; + sc->cmp_offset = 0; sc->flags.disk_io_pending = 0; sc->entry = e; sc->type = storeClientType(e); @@ -158,6 +158,7 @@ storeClientCallback(store_client * sc, ssize_t sz) STCB *callback = sc->callback; char *buf = sc->copy_buf; assert(sc->callback); + sc->cmp_offset = sc->copy_offset + sz; sc->callback = NULL; sc->copy_buf = NULL; if (cbdataValid(sc->callback_data)) @@ -175,11 +176,19 @@ storeClientCopyEvent(void *data) storeClientCopy2(sc->entry, sc); } +void +storeClientCopyOld(store_client *sc, StoreEntry *e, off_t seen_offset, + off_t copy_offset, size_t size, char *buf, STCB *callback, void *data) +{ + /* OLD API -- adrian */ + fatal("storeClientCopyOld() has been called!\n"); +} + + /* copy bytes requested by the client */ void storeClientCopy(store_client * sc, StoreEntry * e, - off_t seen_offset, off_t copy_offset, size_t size, char *buf, @@ -187,9 +196,8 @@ storeClientCopy(store_client * sc, void *data) { assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); - debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", + debug(20, 3) ("storeClientCopy: %s, want %d, size %d, cb %p, cbdata %p\n", storeKeyText(e->hash.key), - (int) seen_offset, (int) copy_offset, (int) size, callback, @@ -200,12 +208,13 @@ storeClientCopy(store_client * sc, #endif assert(sc->callback == NULL); assert(sc->entry == e); + assert(sc->cmp_offset == copy_offset); sc->copy_offset = copy_offset; - sc->seen_offset = seen_offset; sc->callback = callback; sc->copy_buf = buf; sc->copy_size = size; sc->copy_offset = copy_offset; + storeClientCopy2(e, sc); } @@ -268,16 +277,20 @@ storeClientCopy3(StoreEntry * e, store_client * sc) MemObject *mem = e->mem_obj; size_t sz; + debug(33, 5) ("co: %d, hi: %d\n", sc->copy_offset, mem->inmem_hi); + if (storeClientNoMoreToSend(e, sc)) { /* There is no more to send! */ storeClientCallback(sc, 0); return; } - if (e->store_status == STORE_PENDING && sc->seen_offset >= mem->inmem_hi) { - /* client has already seen this, wait for more */ - debug(20, 3) ("storeClientCopy3: Waiting for more\n"); - return; + + /* Check that we actually have data */ + if (e->store_status == STORE_PENDING && sc->copy_offset >= mem->inmem_hi) { + debug(20, 3) ("storeClientCopy3: Waiting for more\n"); + return; } + /* * Slight weirdness here. We open a swapin file for any * STORE_DISK_CLIENT, even if we can copy the requested chunk @@ -289,6 +302,7 @@ storeClientCopy3(StoreEntry * e, store_client * sc) * is clientCacheHit) so that we can fall back to a cache miss * if needed. */ + if (STORE_DISK_CLIENT == sc->type && NULL == sc->swapin_sio) { debug(20, 3) ("storeClientCopy3: Need to open swap in file\n"); /* gotta open the swapin file */ @@ -314,11 +328,12 @@ storeClientCopy3(StoreEntry * e, store_client * sc) } if (sc->copy_offset >= mem->inmem_lo && sc->copy_offset < mem->inmem_hi) { /* What the client wants is in memory */ - debug(20, 3) ("storeClientCopy3: Copying from memory\n"); - sz = stmemCopy(&mem->data_hdr, - sc->copy_offset, sc->copy_buf, sc->copy_size); - storeClientCallback(sc, sz); - return; + /* Old style */ + debug(20, 3) ("storeClientCopy3: Copying normal from memory\n"); + sz = stmemCopy(&mem->data_hdr, sc->copy_offset, sc->copy_buf, + sc->copy_size); + storeClientCallback(sc, sz); + return; } /* What the client wants is not in memory. Schedule a disk read */ assert(STORE_DISK_CLIENT == sc->type); @@ -331,6 +346,7 @@ static void storeClientFileRead(store_client * sc) { MemObject *mem = sc->entry->mem_obj; + assert(sc->callback != NULL); assert(!sc->flags.disk_io_pending); sc->flags.disk_io_pending = 1; diff --git a/src/structs.h b/src/structs.h index 5885ddf021..753c56d9f7 100644 --- a/src/structs.h +++ b/src/structs.h @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.409 2002/02/13 19:34:02 hno Exp $ + * $Id: structs.h,v 1.410 2002/02/26 15:48:16 adrian Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1045,6 +1045,8 @@ struct _clientHttpRequest { request_t *request; /* Parsed URL ... */ store_client *sc; /* The store_client we're using */ store_client *old_sc; /* ... for entry to be validated */ + int old_reqofs; /* ... for the buffer */ + int old_reqsize; /* ... again, for the buffer */ char *uri; char *log_uri; struct { @@ -1076,6 +1078,11 @@ struct _clientHttpRequest { char *location; } redirect; dlink_node active; + char norm_reqbuf[HTTP_REQBUF_SZ]; /* For 'normal requests' */ + char ims_reqbuf[HTTP_REQBUF_SZ]; /* For 'ims' requests */ + char *reqbuf; + int reqofs; + int reqsize; }; struct _ConnStateData { @@ -1165,6 +1172,9 @@ struct _DigestFetchState { int msg; int bytes; } sent, recv; + char buf[SM_PAGE_SIZE]; + int bufofs; + digest_read_state_t state; }; /* statistics for cache digests and other hit "predictors" */ @@ -1413,7 +1423,7 @@ struct _mem_hdr { struct _store_client { int type; off_t copy_offset; - off_t seen_offset; + off_t cmp_offset; size_t copy_size; char *copy_buf; STCB *callback; diff --git a/src/urn.cc b/src/urn.cc index d1d4938fee..51fb68bb16 100644 --- a/src/urn.cc +++ b/src/urn.cc @@ -1,6 +1,6 @@ /* - * $Id: urn.cc,v 1.68 2001/10/24 08:19:09 hno Exp $ + * $Id: urn.cc,v 1.69 2002/02/26 15:48:16 adrian Exp $ * * DEBUG: section 52 URN Parsing * AUTHOR: Kostas Anagnostakis @@ -35,6 +35,8 @@ #include "squid.h" +#define URN_REQBUF_SZ 4096 + typedef struct { StoreEntry *entry; store_client *sc; @@ -44,6 +46,8 @@ typedef struct { struct { unsigned int force_menu:1; } flags; + char reqbuf[URN_REQBUF_SZ]; + int reqofs; } UrnState; typedef struct { @@ -146,11 +150,11 @@ urnStart(request_t * r, StoreEntry * e) } urnState->urlres_e = urlres_e; urnState->urlres_r = requestLink(urlres_r); + urnState->reqofs = 0; storeClientCopy(urnState->sc, urlres_e, 0, - 0, - 4096, - memAllocate(MEM_4K_BUF), + URN_REQBUF_SZ, + urnState->reqbuf, urnHandleReply, urnState); } @@ -171,7 +175,7 @@ url_entry_sort(const void *A, const void *B) } static void -urnHandleReply(void *data, char *buf, ssize_t size) +urnHandleReply(void *data, char *unused_buf, ssize_t size) { UrnState *urnState = data; StoreEntry *e = urnState->entry; @@ -187,35 +191,42 @@ urnHandleReply(void *data, char *buf, ssize_t size) int i; int urlcnt = 0; http_version_t version; + char *buf = urnState->reqbuf; debug(52, 3) ("urnHandleReply: Called with size=%d.\n", (int) size); if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED)) { - memFree(buf, MEM_4K_BUF); - return; + goto error; } if (size == 0) { - memFree(buf, MEM_4K_BUF); - return; + goto error; } else if (size < 0) { - memFree(buf, MEM_4K_BUF); - return; + goto error; } - if (urlres_e->store_status == STORE_PENDING && size < SM_PAGE_SIZE) { + /* Update reqofs to point to where in the buffer we'd be */ + urnState->reqofs += size; + + /* Handle reqofs being bigger than normal */ + if (urnState->reqofs >= URN_REQBUF_SZ) { + goto error; + } + + /* If we haven't received the entire object (urn), copy more */ + if (urlres_e->store_status == STORE_PENDING && + urnState->reqofs < URN_REQBUF_SZ) { storeClientCopy(urnState->sc, urlres_e, - size, - 0, - SM_PAGE_SIZE, - buf, + urnState->reqofs, + URN_REQBUF_SZ, + urnState->reqbuf + urnState->reqofs, urnHandleReply, urnState); return; } /* we know its STORE_OK */ - k = headersEnd(buf, size); + k = headersEnd(buf, urnState->reqofs); if (0 == k) { debug(52, 1) ("urnHandleReply: didn't find end-of-headers for %s\n", storeUrl(e)); - return; + goto error; } s = buf + k; assert(urlres_e->mem_obj->reply); @@ -228,7 +239,7 @@ urnHandleReply(void *data, char *buf, ssize_t size) err->request = requestLink(urnState->request); err->url = xstrdup(storeUrl(e)); errorAppendEntry(e, err); - return; + goto error; } while (xisspace(*s)) s++; @@ -242,7 +253,7 @@ urnHandleReply(void *data, char *buf, ssize_t size) err->request = requestLink(urnState->request); err->url = xstrdup(storeUrl(e)); errorAppendEntry(e, err); - return; + goto error; } min_u = urnFindMinRtt(urls, urnState->request->method, NULL); qsort(urls, urlcnt, sizeof(*urls), url_entry_sort); @@ -285,7 +296,6 @@ urnHandleReply(void *data, char *buf, ssize_t size) httpBodySet(&rep->body, &mb); httpReplySwapOut(rep, e); storeComplete(e); - memFree(buf, MEM_4K_BUF); for (i = 0; i < urlcnt; i++) { safe_free(urls[i].url); safe_free(urls[i].host); @@ -293,6 +303,7 @@ urnHandleReply(void *data, char *buf, ssize_t size) safe_free(urls); /* mb was absorbed in httpBodySet call, so we must not clean it */ storeUnregister(urnState->sc, urlres_e, urnState); +error: storeUnlockObject(urlres_e); storeUnlockObject(urnState->entry); requestUnlink(urnState->request); -- 2.47.3