]> git.ipfire.org Git - thirdparty/squid.git/commitdiff
Added ICAP RESPMOD PRECACHE
authorwessels <>
Tue, 22 Nov 2005 06:29:08 +0000 (06:29 +0000)
committerwessels <>
Tue, 22 Nov 2005 06:29:08 +0000 (06:29 +0000)
These are fairly significant changes to HTTP server-side
processing code to support ICAP request modification.

src/http.cc
src/http.h
src/structs.h

index 19473129ab186e8a9fbf7da95ea26658d4b34854..b5dc30ea6f632606d2125229e78ff6d3cb94ab1a 100644 (file)
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.cc,v 1.466 2005/11/07 22:00:38 wessels Exp $
+ * $Id: http.cc,v 1.467 2005/11/21 23:29:08 wessels Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
 #if DELAY_POOLS
 #include "DelayPools.h"
 #endif
+#if ICAP_CLIENT
+#include "ICAP/ICAPClientRespmodPrecache.h"
+#include "ICAP/ICAPConfig.h"
+#endif
 
-CBDATA_TYPE(HttpStateData);
-
+CBDATA_CLASS_INIT(HttpStateData);
 
 static const char *const crlf = "\r\n";
 
@@ -71,38 +74,62 @@ static void httpMaybeRemovePublic(StoreEntry *, http_status);
 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, String strConnection, HttpRequest * request, HttpRequest * orig_request,
         HttpHeader * hdr_out, int we_do_ranges, http_state_flags);
 static int decideIfWeDoRanges (HttpRequest * orig_request);
+#if ICAP_CLIENT
+static void icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data);
+#endif
 
+HttpStateData::HttpStateData()
+{}
 
-static void
-httpStateFree(int fd, void *data)
+HttpStateData::~HttpStateData()
 {
-    HttpStateData *httpState = static_cast<HttpStateData *>(data);
-
-    if (httpState == NULL)
-        return;
-
-    if (httpState->request_body_buf) {
-        if (httpState->orig_request->body_connection.getRaw()) {
-            clientAbortBody(httpState->orig_request);
+    if (request_body_buf) {
+        if (orig_request->body_connection.getRaw()) {
+            clientAbortBody(orig_request);
         }
 
-        if (httpState->request_body_buf) {
-            memFree(httpState->request_body_buf, MEM_8K_BUF);
-            httpState->request_body_buf = NULL;
+        if (request_body_buf) {
+            memFree(request_body_buf, MEM_8K_BUF);
+            request_body_buf = NULL;
         }
     }
 
-    storeUnlockObject(httpState->entry);
+    storeUnlockObject(entry);
+
+    if (!readBuf->isNull())
+        readBuf->clean();
 
-    if (!httpState->reply_hdr.isNull()) {
-        httpState->reply_hdr.clean();
+    delete readBuf;
+
+    requestUnlink(request);
+
+    requestUnlink(orig_request);
+
+    request = NULL;
+
+    orig_request = NULL;
+
+    if (reply)
+        delete reply;
+
+#if ICAP_CLIENT
+
+    if (icap) {
+        delete icap;
+        cbdataReferenceDone(icap);
     }
 
-    requestUnlink(httpState->request);
-    requestUnlink(httpState->orig_request);
-    httpState->request = NULL;
-    httpState->orig_request = NULL;
-    cbdataFree(httpState);
+#endif
+}
+
+static void
+httpStateFree(int fd, void *data)
+{
+    HttpStateData *httpState = static_cast<HttpStateData *>(data);
+    debug(11,5)("httpStateFree: FD %d, httpState=%p\n", fd, data);
+
+    if (httpState)
+        delete httpState;
 }
 
 int
@@ -342,7 +369,7 @@ HttpStateData::processSurrogateControl(HttpReply *reply)
 int
 HttpStateData::cacheableReply()
 {
-    HttpReply const *rep = entry->getReply();
+    HttpReply const *rep = getReply();
     HttpHeader const *hdr = &rep->header;
     const int cc_mask = (rep->cache_control) ? rep->cache_control->mask : 0;
     const char *v;
@@ -441,7 +468,7 @@ HttpStateData::cacheableReply()
         if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
             return 0;
 
-    switch (entry->getReply()->sline.status) {
+    switch (getReply()->sline.status) {
         /* Responses that are cacheable */
 
     case HTTP_OK:
@@ -634,131 +661,152 @@ HttpStateData::failReply(HttpReply *reply, http_status const & status)
     storeEntryReplaceObject (entry, reply);
 
     if (eof == 1) {
-        fwdComplete(fwd);
-        comm_close(fd);
+        transactionComplete();
     }
 }
 
-/* rewrite this later using new interfaces @?@
+void
+HttpStateData::keepaliveAccounting(HttpReply *reply)
+{
+    if (flags.keepalive)
+        if (_peer)
+            _peer->stats.n_keepalives_sent++;
+
+    if (reply->keep_alive) {
+        if (_peer)
+            _peer->stats.n_keepalives_recv++;
+
+        if (Config.onoff.detect_broken_server_pconns && reply->bodySize(request->method) == -1) {
+            debug(11, 1) ("keepaliveAccounting: Impossible keep-alive header from '%s'\n", storeUrl(entry));
+            // debug(11, 2) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", readBuf->content());
+            flags.keepalive_broken = 1;
+        }
+    }
+}
+
+void
+HttpStateData::checkDateSkew(HttpReply *reply)
+{
+    if (reply->date > -1 && !_peer) {
+        int skew = abs((int)(reply->date - squid_curtime));
+
+        if (skew > 86400)
+            debug(11, 3) ("%s's clock is skewed by %d seconds!\n",
+                          request->host, skew);
+    }
+}
+
+/*
  * This creates the error page itself.. its likely
  * that the forward ported reply header max size patch
  * generates non http conformant error pages - in which
  * case the errors where should be 'BAD_GATEWAY' etc
  */
 void
-HttpStateData::processReplyHeader(const char *buf, int size)
+HttpStateData::processReplyHeader()
 {
-    size_t hdr_len;
-    size_t hdr_size;
     /* Creates a blank header. If this routine is made incremental, this will
      * not do 
      */
-    HttpReply *reply = new HttpReply;
+    reply = new HttpReply;
     Ctx ctx = ctx_enter(entry->mem_obj->url);
-    debug(11, 3) ("processReplyHeader: key '%s'\n",
-                  entry->getMD5Text());
-
-    if (reply_hdr.isNull())
-        reply_hdr.init();
+    debug(11, 3) ("processReplyHeader: key '%s'\n", entry->getMD5Text());
 
     assert(!flags.headers_parsed);
 
-    reply_hdr.append(buf, size);
+    http_status error = HTTP_STATUS_NONE;
 
-    hdr_len = reply_hdr.size;
+    const bool parsed = reply->parse(readBuf, eof, &error);
 
-    if (hdr_len > 4 && strncmp(reply_hdr.buf, "HTTP/", 5)) {
-        debugs(11, 3, "processReplyHeader: Non-HTTP-compliant header: '" <<  reply_hdr.buf << "'");
+    if (!parsed && error > 0) { // unrecoverable parsing error
+        debugs(11, 3, "processReplyHeader: Non-HTTP-compliant header: '" <<  readBuf->content() << "'");
         flags.headers_parsed = 1;
-        reply_hdr.clean();
-        failReply (reply, HTTP_INVALID_HEADER);
+        // negated result yields http_status
+        failReply (reply, error);      // consumes reply
+        reply = NULL;
         ctx_exit(ctx);
         return;
     }
 
-    hdr_size = headersEnd(reply_hdr.buf, hdr_len);
-
-    if (hdr_size)
-        hdr_len = hdr_size;
-
-    if (hdr_len > Config.maxReplyHeaderSize) {
-        debugs(11, 1, "processReplyHeader: Too large reply header");
-
-        if (!reply_hdr.isNull())
-            reply_hdr.clean();
-
-        failReply (reply, HTTP_HEADER_TOO_LARGE);
-
-        flags.headers_parsed = 1;
-
+    if (!parsed) { // need more data
+        assert(!error);
+        assert(!eof);
+        delete reply;
+        reply = NULL;
         ctx_exit(ctx);
-
         return;
     }
 
-    /* headers can be incomplete only if object still arriving */
-    if (!hdr_size) {
-        if (eof)
-            hdr_size = hdr_len;
-        else {
-            delete reply;
-            ctx_exit(ctx);
-            return;            /* headers not complete */
-        }
-    }
-
-    /* Cut away any excess body data (only needed for debug?) */
-    reply_hdr.append("\0", 1);
-
-    reply_hdr.buf[hdr_size] = '\0';
-
+    readBuf->consume(headersEnd(readBuf->content(), readBuf->contentSize()));
     flags.headers_parsed = 1;
 
     debug(11, 9) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n",
-                  reply_hdr.buf);
+                  readBuf->content());
 
-    /* Parse headers into reply structure */
-    /* what happens if we fail to parse here? */
-    reply->parseCharBuf(reply_hdr.buf, hdr_size);
+    keepaliveAccounting(reply);
 
-    if (reply->sline.status >= HTTP_INVALID_HEADER) {
-        debugs(11, 3, "processReplyHeader: Non-HTTP-compliant header: '" << reply_hdr.buf << "'");
-        failReply (reply, HTTP_INVALID_HEADER);
-        ctx_exit(ctx);
-        return;
-    }
+    checkDateSkew(reply);
 
     processSurrogateControl (reply);
-    /* TODO: we need our own reply * in the httpState, as we probably don't want to replace
-     * the storeEntry with interim headers
-     */
 
     /* TODO: IF the reply is a 1.0 reply, AND it has a Connection: Header
      * Parse the header and remove all referenced headers
      */
 
+#if ICAP_CLIENT
+
+    {
+        ICAPAccessCheck *icap_access_check =
+            new ICAPAccessCheck(ICAP::methodRespmod, ICAP::pointPreCache, request, reply, icapAclCheckDoneWrapper, this);
+
+        icapAccessCheckPending = true;
+        icap_access_check->check(); // will eventually delete self
+        ctx_exit(ctx);
+        return;
+    }
+#endif
+
     storeEntryReplaceObject(entry, reply);
-    /* DO NOT USE reply now */
+
+    /* Note storeEntryReplaceObject() consumes reply, so we cannot use it */
     reply = NULL;
 
-    if (entry->getReply()->sline.status == HTTP_PARTIAL_CONTENT &&
-            entry->getReply()->content_range)
-        currentOffset = entry->getReply()->content_range->spec.offset;
+    haveParsedReplyHeaders();
+
+    if (eof == 1) {
+        transactionComplete();
+    }
+
+    ctx_exit(ctx);
+}
+
+/*
+ * This function used to be joined with processReplyHeader(), but
+ * we split it for ICAP.
+ */
+void
+HttpStateData::haveParsedReplyHeaders()
+{
+    Ctx ctx = ctx_enter(entry->mem_obj->url);
+
+    if (getReply()->sline.status == HTTP_PARTIAL_CONTENT &&
+            getReply()->content_range)
+        currentOffset = getReply()->content_range->spec.offset;
 
     storeTimestampsSet(entry);
 
     /* Check if object is cacheable or not based on reply code */
-    debug(11, 3) ("processReplyHeader: HTTP CODE: %d\n", entry->getReply()->sline.status);
+    debug(11, 3) ("haveParsedReplyHeaders: HTTP CODE: %d\n", getReply()->sline.status);
 
     if (neighbors_do_private_keys)
-        httpMaybeRemovePublic(entry, entry->getReply()->sline.status);
+        httpMaybeRemovePublic(entry, getReply()->sline.status);
 
-    if (httpHeaderHas(&entry->getReply()->header, HDR_VARY)
+    if (httpHeaderHas(&getReply()->header, HDR_VARY)
 #if X_ACCELERATOR_VARY
-            || httpHeaderHas(&entry->getReply()->header, HDR_X_ACCELERATOR_VARY)
+            || httpHeaderHas(&getReply()->header, HDR_X_ACCELERATOR_VARY)
 #endif
        ) {
-        const char *vary = httpMakeVaryMark(orig_request, entry->getReply());
+        const char *vary = httpMakeVaryMark(orig_request, getReply());
 
         if (!vary) {
             httpMakePrivate(entry);
@@ -769,6 +817,17 @@ HttpStateData::processReplyHeader(const char *buf, int size)
         entry->mem_obj->vary_headers = xstrdup(vary);
     }
 
+#if WIP_FWD_LOG
+    fwdStatus(fwd, s);
+
+#endif
+    /*
+     * If its not a reply that we will re-forward, then
+     * allow the client to get it.
+     */
+    if (!fwdReforwardableStatus(getReply()->sline.status))
+        EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
+
     switch (cacheableReply()) {
 
     case 1:
@@ -796,55 +855,27 @@ HttpStateData::processReplyHeader(const char *buf, int size)
 
 no_cache:
 
-    if (!ignoreCacheControl && entry->getReply()->cache_control) {
-        if (EBIT_TEST(entry->getReply()->cache_control->mask, CC_PROXY_REVALIDATE))
+    if (!ignoreCacheControl && getReply()->cache_control) {
+        if (EBIT_TEST(getReply()->cache_control->mask, CC_PROXY_REVALIDATE))
             EBIT_SET(entry->flags, ENTRY_REVALIDATE);
-        else if (EBIT_TEST(entry->getReply()->cache_control->mask, CC_MUST_REVALIDATE))
+        else if (EBIT_TEST(getReply()->cache_control->mask, CC_MUST_REVALIDATE))
             EBIT_SET(entry->flags, ENTRY_REVALIDATE);
     }
 
-    if (flags.keepalive)
-        if (_peer)
-            _peer->stats.n_keepalives_sent++;
-
-    if (entry->getReply()->keep_alive) {
-        if (_peer)
-            _peer->stats.n_keepalives_recv++;
-
-        if (Config.onoff.detect_broken_server_pconns && reply->bodySize(request->method) == -1) {
-            debug(11, 1) ("processReplyHeader: Impossible keep-alive header from '%s'\n", storeUrl(entry));
-            debug(11, 2) ("GOT HTTP REPLY HDR:\n---------\n%s\n----------\n", reply_hdr.buf);
-            flags.keepalive_broken = 1;
-        }
-    }
-
-    if (entry->getReply()->date > -1 && !_peer) {
-        int skew = abs((int)(entry->getReply()->date - squid_curtime));
-
-        if (skew > 86400)
-            debug(11, 3) ("%s's clock is skewed by %d seconds!\n",
-                          request->host, skew);
-    }
-
     ctx_exit(ctx);
 #if HEADERS_LOG
 
-    headersLog(1, 0, request->method, entry->getReply());
+    headersLog(1, 0, request->method, getReply());
 #endif
-
-    if (eof == 1) {
-        fwdComplete(fwd);
-        comm_close(fd);
-    }
 }
 
 HttpStateData::ConnectionStatus
 HttpStateData::statusIfComplete() const
 {
-    HttpReply const *reply = entry->getReply();
+    HttpReply const *rep = getReply();
     /* If the reply wants to close the connection, it takes precedence */
 
-    if (httpHeaderHasConnDir(&reply->header, "close"))
+    if (httpHeaderHasConnDir(&rep->header, "close"))
         return COMPLETE_NONPERSISTENT_MSG;
 
     /* If we didn't send a keep-alive request header, then this
@@ -876,7 +907,7 @@ HttpStateData::statusIfComplete() const
      * and an error status code, and we might have to wait until
      * the server times out the socket.
      */
-    if (!reply->keep_alive)
+    if (!rep->keep_alive)
         return COMPLETE_NONPERSISTENT_MSG;
 
     return COMPLETE_PERSISTENT_MSG;
@@ -885,19 +916,23 @@ HttpStateData::statusIfComplete() const
 HttpStateData::ConnectionStatus
 HttpStateData::persistentConnStatus() const
 {
-    HttpReply const *reply = entry->getReply();
+    HttpReply const *reply = getReply();
     int clen;
-    debug(11, 3) ("httpPconnTransferDone: FD %d\n", fd);
+    debug(11, 3) ("persistentConnStatus: FD %d\n", fd);
     ConnectionStatus result = statusIfComplete();
-    debug(11, 5) ("httpPconnTransferDone: content_length=%d\n",
+    debug(11, 5) ("persistentConnStatus: content_length=%d\n",
                   reply->content_length);
     /* If we haven't seen the end of reply headers, we are not done */
 
+    debug(11,5)("persistentConnStatus: flags.headers_parsed=%d\n", flags.headers_parsed);
+
     if (!flags.headers_parsed)
         return INCOMPLETE_MSG;
 
     clen = reply->bodySize(request->method);
 
+    debug(11,5)("persistentConnStatus: clen=%d\n", clen);
+
     /* If there is no message body, we can be persistent */
     if (0 == clen)
         return result;
@@ -907,38 +942,43 @@ HttpStateData::persistentConnStatus() const
         return INCOMPLETE_MSG;
 
     /* If the body size is known, we must wait until we've gotten all of it.  */
-    if (entry->mem_obj->endOffset() < reply->content_length + reply->hdr_sz)
+    /* old technique:
+     * if (entry->mem_obj->endOffset() < reply->content_length + reply->hdr_sz) */
+    debug(11,5)("persistentConnStatus: body_bytes_read=%d, content_length=%d\n",
+                body_bytes_read, reply->content_length);
+
+    if (body_bytes_read < reply->content_length)
         return INCOMPLETE_MSG;
 
     /* We got it all */
     return result;
 }
 
+/*
+ * This is the callback after some data has been read from the network
+ */
 static void
-httpReadReply(int fd, char *buf, size_t len, comm_err_t flag, int xerrno,void *data)
+httpReadReply(int fd, char *buf, size_t len, comm_err_t flag, int xerrno, void *data)
 {
     HttpStateData *httpState = static_cast<HttpStateData *>(data);
     assert (fd == httpState->fd);
+    // assert(buf == readBuf->content());
     PROF_start(HttpStateData_readReply);
-    httpState->readReply (fd, buf, len, flag, xerrno, data);
+    httpState->readReply (len, flag, xerrno);
     PROF_stop(HttpStateData_readReply);
 }
 
-/* This will be called when data is ready to be read from fd.  Read until
- * error or connection closed. */
 /* XXX this function is too long! */
 void
-HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, int xerrno,void *data)
+HttpStateData::readReply (size_t len, comm_err_t flag, int xerrno)
 {
     int bin;
     int clen;
     flags.do_next_read = 0;
 
-
-    assert(buf == readBuf);
-
-    /* Bail out early on COMM_ERR_CLOSING - close handlers will tidy up for us
-    */
+    /*
+     * Bail out early on COMM_ERR_CLOSING - close handlers will tidy up for us
+     */
 
     if (flag == COMM_ERR_CLOSING) {
         debug (11,3)("http socket closing\n");
@@ -952,16 +992,14 @@ HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, in
 
     errno = 0;
     /* prepare the read size for the next read (if any) */
-#if DELAY_POOLS
-
-    DelayId delayId = entry->mem_obj->mostBytesAllowed();
-
-#endif
 
     debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, (int)len);
 
     if (flag == COMM_OK && len > 0) {
+        readBuf->appended(len);
 #if DELAY_POOLS
+
+        DelayId delayId = entry->mem_obj->mostBytesAllowed();
         delayId.bytesIn(len);
 #endif
 
@@ -1012,7 +1050,7 @@ HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, in
             flags.do_next_read = 0;
             comm_close(fd);
         }
-    } else if (flag == COMM_OK && len == 0 && entry->isEmpty()) {
+    } else if (flag == COMM_OK && len == 0 && !flags.headers_parsed) {
         fwdFail(fwd, errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_BAD_GATEWAY));
         eof = 1;
         flags.do_next_read = 0;
@@ -1028,8 +1066,8 @@ HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, in
             * definately at EOF, so we want to process the reply
             * headers.
              */
-            processReplyHeader(buf, len);
-        else if (entry->getReply()->sline.status == HTTP_INVALID_HEADER && HttpVersion(0,9) != entry->getReply()->sline.version) {
+            processReplyHeader();
+        else if (getReply()->sline.status == HTTP_INVALID_HEADER && HttpVersion(0,9) != getReply()->sline.version) {
             fwdFail(fwd, errorCon(ERR_INVALID_RESP, HTTP_BAD_GATEWAY));
             flags.do_next_read = 0;
         } else {
@@ -1037,20 +1075,19 @@ HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, in
                 storeEntryReset(entry);
                 fwdFail(fwd, errorCon(ERR_TOO_BIG, HTTP_BAD_GATEWAY));
                 fwd->flags.dont_retry = 1;
+                flags.do_next_read = 0;
+                comm_close(fd);
             } else {
-                fwdComplete(fwd);
+                transactionComplete();
             }
-
-            flags.do_next_read = 0;
-            comm_close(fd);
         }
     } else {
         if (!flags.headers_parsed) {
-            processReplyHeader(buf, len);
+            processReplyHeader();
 
             if (flags.headers_parsed) {
-                http_status s = entry->getReply()->sline.status;
-                HttpVersion httpver = entry->getReply()->sline.version;
+                http_status s = getReply()->sline.status;
+                HttpVersion httpver = getReply()->sline.version;
 
                 if (s == HTTP_INVALID_HEADER && httpver != HttpVersion(0,9)) {
                     storeEntryReset(entry);
@@ -1059,29 +1096,45 @@ HttpStateData::readReply (int fd, char *readBuf, size_t len, comm_err_t flag, in
                     return;
                 }
 
-#if WIP_FWD_LOG
+            }
+        }
+
+        PROF_start(HttpStateData_processReplyBody);
+        processReplyBody();
+        PROF_stop(HttpStateData_processReplyBody);
+    }
+}
+
+/*
+ * Call this when there is data from the origin server
+ * which should be sent to either StoreEntry, or to ICAP...
+ */
+void
+HttpStateData::writeReplyBody(const char *data, int len)
+{
+#if ICAP_CLIENT
 
-                fwdStatus(fwd, s);
+    if (icap)  {
+        icap->sendMoreData (StoreIOBuffer(len, 0, (char*)data));
+        return;
+    }
 
 #endif
-                /*
-                 * If its not a reply that we will re-forward, then
-                 * allow the client to get it.
-                 */
 
-                if (!fwdReforwardableStatus(s))
-                    EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
-            }
-        }
+    entry->write (StoreIOBuffer(len, currentOffset, (char*)data));
 
-        PROF_start(HttpStateData_processReplyData);
-        processReplyData(buf, len);
-        PROF_stop(HttpStateData_processReplyData);
-    }
+    currentOffset += len;
 }
 
+/*
+ * processReplyBody has two purposes:
+ *  1 - take the reply body data, if any, and put it into either
+ *      the StoreEntry, or give it over to ICAP.
+ *  2 - see if we made it to the end of the response (persistent
+ *      connections and such)
+ */
 void
-HttpStateData::processReplyData(const char *buf, size_t len)
+HttpStateData::processReplyBody()
 {
     if (!flags.headers_parsed) {
         flags.do_next_read = 1;
@@ -1089,36 +1142,26 @@ HttpStateData::processReplyData(const char *buf, size_t len)
         return;
     }
 
-    StoreIOBuffer tempBuffer;
+#if ICAP_CLIENT
+    if (icapAccessCheckPending)
+        return;
 
-    if (!flags.headers_pushed) {
-        /* The first block needs us to skip the headers */
-        /* TODO: make this cleaner. WE should push the headers, NOT the parser */
-        size_t end = headersEnd (buf, len);
-        /* IF len > end, we need to append data after the
-         * out of band update to the store
-         */
+#endif
 
-        if (len > end) {
-            tempBuffer.data = (char *)buf+end;
-            tempBuffer.length = len - end;
-            tempBuffer.offset = currentOffset;
-            currentOffset += tempBuffer.length;
-            entry->write (tempBuffer);
-        }
+    /*
+     * At this point the reply headers have been parsed and consumed.
+     * That means header content has been removed from readBuf and
+     * it contains only body data.
+     */
+    writeReplyBody(readBuf->content(), readBuf->contentSize());
 
-        flags.headers_pushed = 1;
-    } else {
-        tempBuffer.data = (char *)buf;
-        tempBuffer.length = len;
-        tempBuffer.offset = currentOffset;
-        currentOffset += len;
-        entry->write(tempBuffer);
-    }
+    body_bytes_read += readBuf->contentSize();
+
+    readBuf->consume(readBuf->contentSize());
 
     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
         /*
-         * the above entry->write() call could ABORT this entry,
+         * the above writeReplyBody() call could ABORT this entry,
          * in that case, the server FD should already be closed.
          * there's nothing for us to do.
          */
@@ -1127,6 +1170,7 @@ HttpStateData::processReplyData(const char *buf, size_t len)
         switch (persistentConnStatus()) {
 
         case INCOMPLETE_MSG:
+            debug(11,5)("processReplyBody: INCOMPLETE_MSG\n");
             /* Wait for more data or EOF condition */
 
             if (flags.keepalive_broken) {
@@ -1139,6 +1183,7 @@ HttpStateData::processReplyData(const char *buf, size_t len)
             break;
 
         case COMPLETE_PERSISTENT_MSG:
+            debug(11,5)("processReplyBody: COMPLETE_PERSISTENT_MSG\n");
             /* yes we have to clear all these! */
             commSetTimeout(fd, -1, NULL, NULL);
             flags.do_next_read = 0;
@@ -1155,23 +1200,14 @@ HttpStateData::processReplyData(const char *buf, size_t len)
                 pconnPush(fd, request->host, request->port, NULL);
             }
 
-            fwdComplete(fwd);
             fd = -1;
-            httpStateFree(fd, this);
+
+            transactionComplete();
             return;
 
         case COMPLETE_NONPERSISTENT_MSG:
-            /* close the connection ourselves */
-            /* yes - same as for a complete persistent conn here */
-            commSetTimeout(fd, -1, NULL, NULL);
-            commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0);
-            comm_remove_close_handler(fd, httpStateFree, this);
-            fwdUnregister(fd, fwd);
-            fwdComplete(fwd);
-            /* TODO: check that fd is still open here */
-            comm_close (fd);
-            fd = -1;
-            httpStateFree(fd, this);
+            debug(11,5)("processReplyBody: COMPLETE_NONPERSISTENT_MSG\n");
+            transactionComplete();
             return;
         }
 
@@ -1181,14 +1217,50 @@ HttpStateData::processReplyData(const char *buf, size_t len)
 void
 HttpStateData::maybeReadData()
 {
+    int read_sz = readBuf->spaceSize();
+#if ICAP_CLIENT
+
+    if (icap) {
+        /*
+         * Our ICAP message pipes have a finite size limit.  We
+         * should not read more data from the network than will fit
+         * into the pipe buffer.  If totally full, don't register
+         * the read handler at all.  The ICAP side will call our
+         * icapSpaceAvailable() method when it has free space again.
+         */
+        int icap_space = icap->potentialSpaceSize();
+
+        debugs(11,9, "HttpStateData may read up to min(" << icap_space <<
+               ", " << read_sz << ") bytes");
+
+        if (icap_space < read_sz)
+            read_sz = icap_space;
+    }
+
+#endif
+
+    debugs(11,9, "HttpStateData may read up to " << read_sz << " bytes");
+
+    /*
+     * why <2? Because delayAwareRead() won't actually read if
+     * you ask it to read 1 byte.  The delayed read request
+     * just gets re-queued until the client side drains, then
+     * the I/O thread hangs.  Better to not register any read
+     * handler until we get a notification from someone that
+     * its okay to read again.
+     */
+    if (read_sz < 2)
+        return;
+
     if (flags.do_next_read) {
         flags.do_next_read = 0;
-        entry->delayAwareRead(fd, buf, SQUID_TCP_SO_RCVBUF, httpReadReply, this);
+        entry->delayAwareRead(fd, readBuf->space(), read_sz, httpReadReply, this);
     }
 }
 
-/* This will be called when request write is complete. Schedule read of
- * reply. */
+/*
+ * This will be called when request write is complete.
+ */
 void
 HttpStateData::SendComplete(int fd, char *bufnotused, size_t size, comm_err_t errflag, void *data)
 {
@@ -1216,21 +1288,56 @@ HttpStateData::SendComplete(int fd, char *bufnotused, size_t size, comm_err_t er
         fwdFail(httpState->fwd, err);
         comm_close(fd);
         return;
-    } else {
-        /*
-         * Set the read timeout here because it hasn't been set yet.
-         * We only set the read timeout after the request has been
-         * fully written to the server-side.  If we start the timeout
-         * after connection establishment, then we are likely to hit
-         * the timeout for POST/PUT requests that have very large
-         * request bodies.
-         */
-        commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState);
     }
 
+    /*
+     * Set the read timeout here because it hasn't been set yet.
+     * We only set the read timeout after the request has been
+     * fully written to the server-side.  If we start the timeout
+     * after connection establishment, then we are likely to hit
+     * the timeout for POST/PUT requests that have very large
+     * request bodies.
+     */
+    commSetTimeout(fd, Config.Timeout.read, httpTimeout, httpState);
+
     httpState->flags.request_sent = 1;
 }
 
+/*
+ * Calling this function marks the end of the HTTP transaction.
+ * i.e., done talking to the HTTP server.  With ICAP, however, that
+ * does not mean that we're done with HttpStateData and the StoreEntry.
+ * We'll be expecting adapted data to come back from the ICAP
+ * routines.
+ */
+void
+HttpStateData::transactionComplete()
+{
+    if (fd >= 0) {
+        fwdUnregister(fd, fwd);
+        comm_remove_close_handler(fd, httpStateFree, this);
+        comm_close(fd);
+        fd = -1;
+    }
+
+#if ICAP_CLIENT
+    if (icap) {
+        icap->doneSending();
+    } else {
+
+#endif
+
+        fwdComplete(fwd);
+
+        httpStateFree(-1, this);
+
+#if ICAP_CLIENT
+
+    }
+
+#endif
+}
+
 /*
  * build request headers and append them to a given MemBuf 
  * used by httpBuildRequestPrefix()
@@ -1275,7 +1382,6 @@ httpBuildRequestHeader(HttpRequest * request,
         orig_request->flags.range = 0;
     }
 
-
     /* append Via */
     if (Config.onoff.via) {
         String strVia;
@@ -1454,7 +1560,6 @@ httpBuildRequestHeader(HttpRequest * request,
     strConnection.clean();
 }
 
-
 void
 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, String strConnection, HttpRequest * request, HttpRequest * orig_request, HttpHeader * hdr_out, int we_do_ranges, http_state_flags flags)
 {
@@ -1607,7 +1712,6 @@ decideIfWeDoRanges (HttpRequest * orig_request)
     return result;
 }
 
-
 /* build request prefix and append it to a given MemBuf;
  * return the length of the prefix */
 mb_size_t
@@ -1652,9 +1756,9 @@ httpSendRequest(HttpStateData * httpState)
     debug(11, 5) ("httpSendRequest: FD %d: httpState %p.\n", fd,
                   httpState);
 
-    /* Schedule read reply. */
     commSetTimeout(fd, Config.Timeout.lifetime, httpTimeout, httpState);
-    entry->delayAwareRead(fd, httpState->buf, SQUID_TCP_SO_RCVBUF, httpReadReply, httpState);
+    httpState->flags.do_next_read = 1;
+    httpState->maybeReadData();
 
     if (httpState->orig_request->body_connection.getRaw() != NULL)
         sendHeaderDone = httpSendRequestEntity;
@@ -1715,14 +1819,15 @@ httpStart(FwdState * fwd)
     debug(11, 3) ("httpStart: \"%s %s\"\n",
                   RequestMethodStr[orig_req->method],
                   storeUrl(fwd->entry));
-    CBDATA_INIT_TYPE(HttpStateData);
-    httpState = cbdataAlloc(HttpStateData);
+    httpState = new HttpStateData;
     httpState->ignoreCacheControl = false;
     httpState->surrogateNoStore = false;
     storeLockObject(fwd->entry);
     httpState->fwd = fwd;
     httpState->entry = fwd->entry;
     httpState->fd = fd;
+    httpState->readBuf = new MemBuf;
+    httpState->readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 
     if (fwd->servers)
         httpState->_peer = fwd->servers->_peer;                /* might be NULL */
@@ -1827,7 +1932,7 @@ httpRequestBodyHandler(char *buf, ssize_t size, void *data)
                           inet_ntoa(httpState->orig_request->client_addr),
                           storeUrl(httpState->entry));
 
-            if (httpState->entry->getReply()->sline.status == HTTP_INVALID_HEADER) {
+            if (httpState->getReply()->sline.status == HTTP_INVALID_HEADER) {
                 memFree8K(buf);
                 comm_close(httpState->fd);
                 return;
@@ -1887,3 +1992,157 @@ httpBuildVersion(HttpVersion * version, unsigned int major, unsigned int minor)
     version->major = major;
     version->minor = minor;
 }
+
+#if ICAP_CLIENT
+
+static void
+icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data)
+{
+    HttpStateData *http = (HttpStateData *)data;
+    http->icapAclCheckDone(service);
+}
+
+void
+HttpStateData::icapAclCheckDone(ICAPServiceRep::Pointer service)
+{
+    icapAccessCheckPending = false;
+
+    if (service == NULL) {
+        // handle case where no service is selected;
+        storeEntryReplaceObject(entry, reply);
+
+        /* Note storeEntryReplaceObject() consumes reply, so we cannot use it */
+        reply = NULL;
+
+        haveParsedReplyHeaders();
+        processReplyBody();
+
+        if (eof == 1)
+            transactionComplete();
+
+        return;
+    }
+
+    if (doIcap(service) < 0) {
+        /*
+         * XXX Maybe instead of an error page we should
+         * handle the reply normally (without ICAP).
+         */
+        ErrorState *err = errorCon(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR);
+        err->xerrno = errno;
+        err->request = requestLink(orig_request);
+        errorAppendEntry(entry, err);
+        comm_close(fd);
+        return;
+    }
+
+    icap->startRespMod(this, request, reply);
+    processReplyBody();
+}
+
+/*
+ * Initiate an ICAP transaction.  Return 0 if all is well, or -1 upon error.
+ * Caller will handle error condition by generating a Squid error message
+ * or take other action.
+ */
+int
+HttpStateData::doIcap(ICAPServiceRep::Pointer service)
+{
+    debug(11,5)("HttpStateData::doIcap() called\n");
+    assert(NULL == icap);
+    icap = new ICAPClientRespmodPrecache(service);
+    (void) cbdataReference(icap);
+    return 0;
+}
+
+/*
+ * Called by ICAPClientRespmodPrecache when it has space available for us.
+ */
+void
+HttpStateData::icapSpaceAvailable()
+{
+    debug(11,5)("HttpStateData::icapSpaceAvailable() called\n");
+    maybeReadData();
+}
+
+void
+HttpStateData::takeAdaptedHeaders(HttpReply *rep)
+{
+    debug(11,5)("HttpStateData::takeAdaptedHeaders() called\n");
+
+    if (!entry->isAccepting()) {
+        debug(11,5)("\toops, entry is not Accepting!\n");
+        icap->ownerAbort();
+        return;
+    }
+
+    storeEntryReplaceObject(entry, rep);
+
+    /*
+     * After calling storeEntryReplaceObject() we give up control
+     * of the rep and this->reply pointers.
+     */
+    rep = NULL;
+
+    haveParsedReplyHeaders();
+
+    debug(11,5)("HttpStateData::takeAdaptedHeaders() finished\n");
+}
+
+void
+HttpStateData::takeAdaptedBody(MemBuf *buf)
+{
+    debug(11,5)("HttpStateData::takeAdaptedBody() called\n");
+    debug(11,5)("\t%d bytes\n", buf->contentSize());
+    debug(11,5)("\t%d is current offset\n", (int)currentOffset);
+
+    if (!entry->isAccepting()) {
+        debug(11,5)("\toops, entry is not Accepting!\n");
+        icap->ownerAbort();
+        return;
+    }
+
+    entry->write(StoreIOBuffer(buf, currentOffset)); // write everything
+    currentOffset += buf->contentSize();
+    buf->consume(buf->contentSize()); // consume everything written
+}
+
+void
+HttpStateData::doneAdapting()
+{
+    debug(11,5)("HttpStateData::doneAdapting() called\n");
+
+    if (!entry->isAccepting()) {
+        debug(11,5)("\toops, entry is not Accepting!\n");
+        icap->ownerAbort();
+    } else {
+        fwdComplete(fwd);
+    }
+
+    assert(fd == -1);
+    httpStateFree(-1, this);
+}
+
+void
+HttpStateData::abortAdapting()
+{
+    debug(11,5)("HttpStateData::abortAdapting() called\n");
+
+    if (entry->isEmpty()) {
+        ErrorState *err;
+        err = errorCon(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR);
+        err->request = requestLink((HttpRequest *) request);
+        err->xerrno = errno;
+        fwdFail(fwd, err);
+        flags.do_next_read = 0;
+    }
+
+    fwdComplete(fwd);
+
+    if (fd >= 0)
+        comm_close(fd);
+    else
+        httpStateFree(fd, this);
+}
+
+#endif
index d7365ee624122ff7fa9beac9250abfb69c5da4b0..9ee2d0950d7e5b2323c773a2b568684bb8a50dc3 100644 (file)
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.h,v 1.14 2005/09/28 20:26:27 wessels Exp $
+ * $Id: http.h,v 1.15 2005/11/21 23:29:08 wessels Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
 #include "StoreIOBuffer.h"
 #include "comm.h"
 
+#if ICAP_CLIENT
+#include "ICAP/ICAPServiceRep.h"
+
+class ICAPClientRespmodPrecache;
+
+class ICAPAccessCheck;
+#endif
+
 class HttpStateData
 {
 
 public:
+    HttpStateData();
+    ~HttpStateData();
+
     static CWCB SendComplete;
     /* should be private */
-    void processReplyHeader(const char *buf, int size);
-    void processReplyData(const char *, size_t);
-    IOCB readReply;
+    void processReplyHeader();
+    void processReplyBody();
+    void readReply(size_t len, comm_err_t flag, int xerrno);
     void maybeReadData();
     int cacheableReply();
+#if ICAP_CLIENT
+
+    void takeAdaptedHeaders(HttpReply *);
+    void takeAdaptedBody(MemBuf *);
+    void doneAdapting();
+    void abortAdapting();
+    void icapSpaceAvailable();
+#endif
 
     StoreEntry *entry;
     HttpRequest *request;
-    MemBuf reply_hdr;
     peer *_peer;               /* peer request made to */
     int eof;                   /* reached end-of-object? */
     HttpRequest *orig_request;
@@ -61,12 +79,32 @@ public:
     char *request_body_buf;
     off_t currentOffset;
     size_t read_sz;
-    char buf[SQUID_TCP_SO_RCVBUF];
+    int body_bytes_read;       /* to find end of response, independent of StoreEntry */
+    MemBuf *readBuf;
     bool ignoreCacheControl;
     bool surrogateNoStore;
     void processSurrogateControl(HttpReply *);
+#if ICAP_CLIENT
+
+    ICAPClientRespmodPrecache *icap;
+    void icapAclCheckDone(ICAPServiceRep::Pointer);
+    bool icapAccessCheckPending;
+#endif
+
+    /*
+     * getReply() public only because it is called from a static function
+     * as httpState->getReply()
+     */
+const HttpReply * getReply() const { return reply ? reply : entry->getReply(); }
 
 private:
+    /*
+     * This HttpReply will be owned by HttpStateData until it is given to the
+     * StoreEntry.  This is necessary/usefulr for ESI/ICAP.  Use this class' getReply()
+     * method to get the reply either directly from this class or from the StoreEntry
+     */
+    HttpReply *reply;
+
     enum ConnectionStatus {
         INCOMPLETE_MSG,
         COMPLETE_PERSISTENT_MSG,
@@ -75,6 +113,18 @@ private:
     ConnectionStatus statusIfComplete() const;
     ConnectionStatus persistentConnStatus() const;
     void failReply (HttpReply *reply, http_status const &status);
+    void keepaliveAccounting(HttpReply *);
+    void checkDateSkew(HttpReply *);
+    void haveParsedReplyHeaders();
+    void transactionComplete();
+    void writeReplyBody(const char *data, int len);
+#if ICAP_CLIENT
+
+    int doIcap(ICAPServiceRep::Pointer);
+#endif
+
+private:
+    CBDATA_CLASS2(HttpStateData);
 };
 
 #endif /* SQUID_HTTP_H */
index 12382b7704a105971dc481dd166913e15506e27f..4c349bd3af4abbd481241adcc76702aba3fbbdc4 100644 (file)
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.531 2005/11/04 20:27:31 wessels Exp $
+ * $Id: structs.h,v 1.532 2005/11/21 23:29:08 wessels Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -971,9 +971,6 @@ unsigned int only_if_cached:
 unsigned int headers_parsed:
     1;
 
-unsigned int headers_pushed:
-    1;
-
 unsigned int front_end_https:
     2;