From: Alex Rousskov Date: Sun, 12 Jul 2009 22:56:47 +0000 (-0600) Subject: Bug #2459 fix, access logging enhancements, ICAP logging and retries support: X-Git-Tag: SQUID_3_2_0_1~886 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3ff6559699bc1852ae3a6f7be48824d13e2ebf90;p=thirdparty%2Fsquid.git Bug #2459 fix, access logging enhancements, ICAP logging and retries support: - Bug #2459 fix - Support logging of total DNS wait time to access.log (%dt) - Support logging response times of adaptation transactions to access.log (%adapt::sum_trs and %adapt::all_trs) - Enhanced access logging (sh, 0) + os << "lookup_wait=" << wait; + if (error.size()) + os << " lookup_err=" << error; + return os; +} diff --git a/src/DnsLookupDetails.h b/src/DnsLookupDetails.h new file mode 100644 index 0000000000..29c13bfbb8 --- /dev/null +++ b/src/DnsLookupDetails.h @@ -0,0 +1,31 @@ +/* + * DEBUG: section 78 DNS lookups + */ + +#ifndef SQUID_DNS_LOOKUP_DETAILS_H +#define SQUID_DNS_LOOKUP_DETAILS_H + +#include "typedefs.h" +#include "SquidString.h" + +/// encapsulates DNS lookup results +class DnsLookupDetails +{ +public: + DnsLookupDetails(); ///< no error, no lookup delay (i.e., no lookup) + DnsLookupDetails(const String &error, int wait); + + std::ostream &print(std::ostream &os) const; + +public: + String error; ///< error message for unsuccessdul lookups; empty otherwise + int wait; ///< msecs spent waiting for the lookup (if any) or -1 (if none) +}; + +inline +std::ostream &operator << (std::ostream &os, const DnsLookupDetails &dns) +{ + return dns.print(os); +} + +#endif diff --git a/src/HierarchyLogEntry.h b/src/HierarchyLogEntry.h index 6aec149471..cb18a27519 100644 --- a/src/HierarchyLogEntry.h +++ b/src/HierarchyLogEntry.h @@ -56,6 +56,12 @@ public: struct timeval peer_select_start; struct timeval store_complete_stop; + + http_status peer_reply_status; ///< last HTTP status code received + timeval peer_http_request_sent; ///< last peer finished writing req + int64_t peer_response_time; ///< last peer response delay + timeval first_conn_start; ///< first connection use among all peers + int64_t total_response_time; ///< cumulative for all peers }; extern void hierarchyNote(HierarchyLogEntry *, hier_code, const char *); diff --git a/src/HttpMsg.cc b/src/HttpMsg.cc index 2aab365654..22e61db892 100644 --- a/src/HttpMsg.cc +++ b/src/HttpMsg.cc @@ -309,6 +309,14 @@ HttpMsg::httpMsgParseError() return -1; } +void +HttpMsg::setContentLength(int64_t clen) +{ + header.delById(HDR_CONTENT_LENGTH); // if any + header.putInt64(HDR_CONTENT_LENGTH, clen); + content_length = clen; +} + /* returns true if connection should be "persistent" * after processing this message */ int diff --git a/src/HttpMsg.h b/src/HttpMsg.h index 4fd7fe1a38..c21c3c1241 100644 --- a/src/HttpMsg.h +++ b/src/HttpMsg.h @@ -58,6 +58,9 @@ public: ///< produce a message copy, except for a few connection-specific settings virtual HttpMsg *clone() const = 0; ///< \todo rename: not a true copy? + /// [re]sets Content-Length header and cached value + void setContentLength(int64_t clen); + public: HttpVersion http_ver; @@ -109,7 +112,8 @@ protected: }; /* Temporary parsing state; might turn into the replacement parser later on */ -struct _HttpParser { +class HttpParser { +public: char state; const char *buf; int bufsiz; @@ -120,7 +124,6 @@ struct _HttpParser { int v_start, v_end; int v_maj, v_min; }; -typedef struct _HttpParser HttpParser; extern void HttpParserInit(HttpParser *, const char *buf, int len); extern int HttpParserParseReqLine(HttpParser *hp); diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 1f80874d41..18fdc731a9 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -40,6 +40,9 @@ #include "HttpHeaderRange.h" #include "MemBuf.h" #include "Store.h" +#if ICAP_CLIENT +#include "adaptation/icap/icap_log.h" +#endif HttpRequest::HttpRequest() : HttpMsg(hoRequest) { @@ -87,6 +90,7 @@ HttpRequest::init() my_addr.SetEmpty(); body_pipe = NULL; // hier + dnsWait = -1; errType = ERR_NONE; peer_login = NULL; // not allocated/deallocated by this class peer_domain = NULL; // not allocated/deallocated by this class @@ -99,6 +103,12 @@ HttpRequest::init() #if FOLLOW_X_FORWARDED_FOR indirect_client_addr.SetEmpty(); #endif /* FOLLOW_X_FORWARDED_FOR */ +#if USE_ADAPTATION + adaptHistory_ = NULL; +#endif +#if ICAP_CLIENT + icapHistory_ = NULL; +#endif } void @@ -138,6 +148,13 @@ HttpRequest::clean() extacl_passwd.clean(); extacl_log.clean(); + +#if USE_ADAPTATION + adaptHistory_ = NULL; +#endif +#if ICAP_CLIENT + icapHistory_ = NULL; +#endif } void @@ -163,25 +180,14 @@ HttpRequest::clone() const strncpy(copy->host, host, sizeof(host)); // SQUIDHOSTNAMELEN copy->host_addr = host_addr; - if (auth_user_request) { - copy->auth_user_request = auth_user_request; - AUTHUSERREQUESTLOCK(copy->auth_user_request, "HttpRequest::clone"); - } - copy->port = port; // urlPath handled in ctor copy->canonical = canonical ? xstrdup(canonical) : NULL; - // This may be too conservative for the 204 No Content case - // may eventually need cloneNullAdaptationImmune() for that. - copy->flags = flags.cloneAdaptationImmune(); - copy->range = range ? new HttpHdrRange(*range) : NULL; copy->ims = ims; copy->imslen = imslen; copy->max_forwards = max_forwards; - copy->client_addr = client_addr; - copy->my_addr = my_addr; copy->hier = hier; // Is it safe to copy? Should we? copy->errType = errType; @@ -197,6 +203,8 @@ HttpRequest::clone() const copy->extacl_passwd = extacl_passwd; copy->extacl_log = extacl_log; + assert(copy->inheritProperties(this)); + return copy; } @@ -356,6 +364,37 @@ request_flags::clearResetTCP() reset_tcp = 0; } +#if ICAP_CLIENT +Adaptation::Icap::History::Pointer +HttpRequest::icapHistory() const +{ + if (!icapHistory_) { + if ((LogfileStatus == LOG_ENABLE && alLogformatHasIcapToken) || + IcapLogfileStatus == LOG_ENABLE) { + icapHistory_ = new Adaptation::Icap::History(); + debugs(93,4, HERE << "made " << icapHistory_ << " for " << this); + } + } + + return icapHistory_; +} +#endif + +#if USE_ADAPTATION +Adaptation::History::Pointer +HttpRequest::adaptHistory() const +{ + if (!adaptHistory_) { + if (Adaptation::History::Enabled) { + adaptHistory_ = new Adaptation::History(); + debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this); + } + } + + return adaptHistory_; +} +#endif + bool HttpRequest::multipartRangeRequest() const { @@ -509,6 +548,15 @@ bool HttpRequest::inheritProperties(const HttpMsg *aMsg) client_addr = aReq->client_addr; my_addr = aReq->my_addr; + dnsWait = aReq->dnsWait; + +#if USE_ADAPTATION + adaptHistory_ = aReq->adaptHistory(); +#endif +#if ICAP_CLIENT + icapHistory_ = aReq->icapHistory(); +#endif + // This may be too conservative for the 204 No Content case // may eventually need cloneNullAdaptationImmune() for that. flags = aReq->flags.cloneAdaptationImmune(); @@ -523,3 +571,13 @@ bool HttpRequest::inheritProperties(const HttpMsg *aMsg) } return true; } + +void HttpRequest::recordLookup(const DnsLookupDetails &dns) +{ + if (dns.wait >= 0) { // known delay + if (dnsWait >= 0) // have recorded DNS wait before + dnsWait += dns.wait; + else + dnsWait = dns.wait; + } +} diff --git a/src/HttpRequest.h b/src/HttpRequest.h index d7981eefa3..2f4407e3ce 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -38,13 +38,28 @@ #include "client_side.h" #include "HierarchyLogEntry.h" #include "HttpRequestMethod.h" +#if USE_ADAPTATION +#include "adaptation/History.h" +#endif +#if ICAP_CLIENT +#include "adaptation/icap/History.h" +#endif /* Http Request */ //DEAD?: extern int httpRequestHdrAllowedByName(http_hdr_type id); extern void httpRequestPack(void *obj, Packer *p); +// TODO: Move these three to access_log.h or AccessLogEntry.h +#if USE_ADAPTATION +extern bool alLogformatHasAdaptToken; +#endif +#if ICAP_CLIENT +extern bool alLogformatHasIcapToken; +#endif +extern int LogfileStatus; class HttpHdrRange; +class DnsLookupDetails; class HttpRequest: public HttpMsg { @@ -84,6 +99,17 @@ public: }; inline const char* GetHost(void) const { return host; }; +#if USE_ADAPTATION + /// Returns possibly nil history, creating it if adapt. logging is enabled + Adaptation::History::Pointer adaptHistory() const; +#endif +#if ICAP_CLIENT + /// Returns possibly nil history, creating it if icap logging is enabled + Adaptation::Icap::History::Pointer icapHistory() const; +#endif + + void recordLookup(const DnsLookupDetails &detail); + protected: void clean(); @@ -103,6 +129,13 @@ private: */ ConnStateData *pinned_connection; +#if USE_ADAPTATION + mutable Adaptation::History::Pointer adaptHistory_; ///< per-HTTP transaction info +#endif +#if ICAP_CLIENT + mutable Adaptation::Icap::History::Pointer icapHistory_; ///< per-HTTP transaction info +#endif + public: IpAddress host_addr; @@ -134,6 +167,8 @@ public: HierarchyLogEntry hier; + int dnsWait; ///< sum of DNS lookup delays in milliseconds, for %dt + err_type errType; char *peer_login; /* Configured peer login:password */ diff --git a/src/Makefile.am b/src/Makefile.am index 1660ba939e..9c77822e3a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,6 +15,10 @@ DNSSOURCE = dns_internal.cc DNSSERVER = endif +DNSSOURCE += \ + DnsLookupDetails.h \ + DnsLookupDetails.cc + SNMP_ALL_SOURCE = \ snmp_core.cc \ snmp_agent.cc @@ -233,6 +237,8 @@ EXTRA_squid_SOURCES = \ dns.cc \ dnsserver.cc \ dns_internal.cc \ + DnsLookupDetails.cc \ + DnsLookupDetails.h \ htcp.cc \ htcp.h \ ipc.cc \ diff --git a/src/Server.cc b/src/Server.cc index b3927970a4..9ea7095755 100644 --- a/src/Server.cc +++ b/src/Server.cc @@ -39,6 +39,7 @@ #include "HttpReply.h" #include "TextException.h" #include "errorpage.h" +#include "SquidTime.h" #if USE_ADAPTATION #include "adaptation/AccessCheck.h" @@ -173,6 +174,10 @@ ServerStateData::serverComplete() completed = true; + HttpRequest *r = originalRequest(); + r->hier.total_response_time = r->hier.first_conn_start.tv_sec ? + tvSubMsec(r->hier.first_conn_start, current_time) : -1; + if (requestBodySource != NULL) stopConsumingFrom(requestBodySource); diff --git a/src/access_log.cc b/src/access_log.cc index 9b7525e2cf..2630214a48 100644 --- a/src/access_log.cc +++ b/src/access_log.cc @@ -106,7 +106,16 @@ static void fvdbClear(void); static void fvdbRegisterWithCacheManager(); #endif -static int LogfileStatus = LOG_DISABLE; +int LogfileStatus = LOG_DISABLE; + +#if USE_ADAPTATION +bool alLogformatHasAdaptToken = false; +#endif + +#if ICAP_CLIENT +bool alLogformatHasIcapToken = false; +#endif + #define LOG_BUF_SZ (MAX_URL<<2) static const char c2x[] = @@ -333,6 +342,10 @@ typedef enum { LFT_TIME_GMT, LFT_TIME_TO_HANDLE_REQUEST, + LFT_PEER_RESPONSE_TIME, + LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, + LFT_DNS_WAIT_TIME, + LFT_REQUEST_HEADER, LFT_REQUEST_HEADER_ELEM, LFT_REQUEST_ALL_HEADERS, @@ -348,7 +361,9 @@ typedef enum { /*LFT_USER_SCHEME, */ LFT_USER_EXTERNAL, - LFT_HTTP_CODE, + LFT_HTTP_SENT_STATUS_CODE_OLD_30, + LFT_HTTP_SENT_STATUS_CODE, + LFT_HTTP_RECEIVED_STATUS_CODE, /*LFT_HTTP_STATUS, */ LFT_SQUID_STATUS, @@ -365,7 +380,7 @@ typedef enum { LFT_REQUEST_SIZE_TOTAL, /*LFT_REQUEST_SIZE_LINE, */ - /*LFT_REQUEST_SIZE_HEADERS, */ + LFT_REQUEST_SIZE_HEADERS, /*LFT_REQUEST_SIZE_BODY, */ /*LFT_REQUEST_SIZE_BODY_NO_TE, */ @@ -373,7 +388,7 @@ typedef enum { LFT_REPLY_HIGHOFFSET, LFT_REPLY_OBJECTSIZE, /*LFT_REPLY_SIZE_LINE, */ - /*LFT_REPLY_SIZE_HEADERS, */ + LFT_REPLY_SIZE_HEADERS, /*LFT_REPLY_SIZE_BODY, */ /*LFT_REPLY_SIZE_BODY_NO_TE, */ @@ -381,6 +396,39 @@ typedef enum { LFT_IO_SIZE_TOTAL, LFT_EXT_LOG, +#if USE_ADAPTATION + LTF_ADAPTATION_SUM_XACT_TIMES, + LTF_ADAPTATION_ALL_XACT_TIMES, +#endif + +#if ICAP_CLIENT + + LFT_ICAP_TOTAL_TIME, + LFT_ICAP_LAST_MATCHED_HEADER, + LFT_ICAP_LAST_MATCHED_HEADER_ELEM, + LFT_ICAP_LAST_MATCHED_ALL_HEADERS, + + LFT_ICAP_ADDR, + LFT_ICAP_SERV_NAME, + LFT_ICAP_REQUEST_URI, + LFT_ICAP_REQUEST_METHOD, + LFT_ICAP_BYTES_SENT, + LFT_ICAP_BYTES_READ, + + LFT_ICAP_REQ_HEADER, + LFT_ICAP_REQ_HEADER_ELEM, + LFT_ICAP_REQ_ALL_HEADERS, + + LFT_ICAP_REP_HEADER, + LFT_ICAP_REP_HEADER_ELEM, + LFT_ICAP_REP_ALL_HEADERS, + + LFT_ICAP_TR_RESPONSE_TIME, + LFT_ICAP_IO_TIME, + LFT_ICAP_OUTCOME, + LFT_ICAP_STATUS_CODE, +#endif + LFT_PERCENT /* special string cases for escaped chars */ } logformat_bcode_t; @@ -447,6 +495,10 @@ struct logformat_token_table_entry logformat_token_table[] = { {"tg", LFT_TIME_GMT}, {"tr", LFT_TIME_TO_HANDLE_REQUEST}, + {"h", LFT_REQUEST_HEADER}, {">h", LFT_REQUEST_ALL_HEADERS}, {"Hs", LFT_HTTP_SENT_STATUS_CODE}, + {"st", LFT_REQUEST_SIZE_TOTAL }, /*{ ">sl", LFT_REQUEST_SIZE_LINE }, * / / * the request line "GET ... " */ - /*{ ">sh", LFT_REQUEST_SIZE_HEADERS }, */ + { ">sh", LFT_REQUEST_SIZE_HEADERS }, /*{ ">sb", LFT_REQUEST_SIZE_BODY }, */ /*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */ @@ -485,7 +539,7 @@ struct logformat_token_table_entry logformat_token_table[] = { {"st", LFT_ICAP_BYTES_SENT}, + {"icap::h", LFT_ICAP_REQ_HEADER}, + {"icap::cache.caddr.IsNoAddr()) // e.g., ICAP OPTIONS lack client + out = "-"; + else out = al->cache.caddr.NtoA(tmp,1024); - } break; case LFT_CLIENT_FQDN: + if (al->cache.caddr.IsAnyAddr()) // e.g., ICAP OPTIONS lack client + out = "-"; + else out = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); if (!out) { out = al->cache.caddr.NtoA(tmp,1024); @@ -620,6 +703,31 @@ accessLogCustom(AccessLogEntry * al, customlog * log) doint = 1; break; + case LFT_PEER_RESPONSE_TIME: + if (al->hier.peer_response_time < 0) { + out = "-"; + } else { + outoff = al->hier.peer_response_time; + dooff = 1; + } + break; + + case LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME: + if (al->hier.total_response_time < 0) { + out = "-"; + } else { + outoff = al->hier.total_response_time; + dooff = 1; + } + break; + + case LFT_DNS_WAIT_TIME: + if (al->request && al->request->dnsWait >= 0) { + outint = al->request->dnsWait; + doint = 1; + } + break; + case LFT_REQUEST_HEADER: if (al->request) @@ -641,6 +749,175 @@ accessLogCustom(AccessLogEntry * al, customlog * log) break; +#if USE_ADAPTATION + case LTF_ADAPTATION_SUM_XACT_TIMES: + if (al->request) { + Adaptation::History::Pointer ah = al->request->adaptHistory(); + if (ah != NULL) + ah->sumLogString(fmt->data.string, sb); + out = sb.termedBuf(); + } + break; + + case LTF_ADAPTATION_ALL_XACT_TIMES: + if (al->request) { + Adaptation::History::Pointer ah = al->request->adaptHistory(); + if (ah != NULL) + ah->allLogString(fmt->data.string, sb); + out = sb.termedBuf(); + } + break; +#endif + +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + if (al->request) { + Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); + if (ih != NULL) + sb = ih->mergeOfIcapHeaders.getByName(fmt->data.header.header); + } + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: + if (al->request) { + Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); + if (ih != NULL) + sb = ih->mergeOfIcapHeaders.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + } + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: + out = al->headers.icap; + + quote = 1; + + break; + + case LFT_ICAP_ADDR: + if (!out) + out = al->icap.hostAddr.NtoA(tmp,1024); + break; + + case LFT_ICAP_SERV_NAME: + out = al->icap.serviceName.termedBuf(); + break; + + case LFT_ICAP_REQUEST_URI: + out = al->icap.reqUri.termedBuf(); + break; + + case LFT_ICAP_REQUEST_METHOD: + out = Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod); + break; + + case LFT_ICAP_BYTES_SENT: + outint = al->icap.bytesSent; + doint = 1; + break; + + case LFT_ICAP_BYTES_READ: + outint = al->icap.bytesRead; + doint = 1; + break; + + case LFT_ICAP_REQ_HEADER: + if (NULL != al->icap.request) { + sb = al->icap.request->header.getByName(fmt->data.header.header); + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REQ_HEADER_ELEM: + if (al->request) + sb = al->icap.request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_REQ_ALL_HEADERS: + if (al->icap.request) { + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = al->icap.request->header.getEntry(&pos)) { + sb.append(e->name); + sb.append(": "); + sb.append(e->value); + sb.append("\r\n"); + } + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REP_HEADER: + if (NULL != al->icap.reply) { + sb = al->icap.reply->header.getByName(fmt->data.header.header); + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REP_HEADER_ELEM: + if (NULL != al->icap.reply) + sb = al->icap.reply->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_REP_ALL_HEADERS: + if (al->icap.reply) { + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = al->icap.reply->header.getEntry(&pos)) { + sb.append(e->name); + sb.append(": "); + sb.append(e->value); + sb.append("\r\n"); + } + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_TR_RESPONSE_TIME: + outint = al->icap.trTime; + doint = 1; + break; + + case LFT_ICAP_IO_TIME: + outint = al->icap.ioTime; + doint = 1; + break; + + case LFT_ICAP_STATUS_CODE: + outint = al->icap.resStatus; + doint = 1; + break; + + case LFT_ICAP_OUTCOME: + out = al->icap.outcome; + break; + + case LFT_ICAP_TOTAL_TIME: + outint = al->icap.processingTime; + doint = 1; + break; +#endif case LFT_REQUEST_HEADER_ELEM: if (al->request) sb = al->request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); @@ -719,13 +996,25 @@ accessLogCustom(AccessLogEntry * al, customlog * log) /* case LFT_USER_REALM: */ /* case LFT_USER_SCHEME: */ - case LFT_HTTP_CODE: + // the fmt->type can not be LFT_HTTP_SENT_STATUS_CODE_OLD_30 + // but compiler complains if ommited + case LFT_HTTP_SENT_STATUS_CODE_OLD_30: + case LFT_HTTP_SENT_STATUS_CODE: outint = al->http.code; doint = 1; break; + case LFT_HTTP_RECEIVED_STATUS_CODE: + if(al->hier.peer_reply_status == HTTP_STATUS_NONE) { + out = "-"; + } + else { + outint = al->hier.peer_reply_status; + doint = 1; + } + break; /* case LFT_HTTP_STATUS: * out = statusline->text; * quote = 1; @@ -780,7 +1069,10 @@ accessLogCustom(AccessLogEntry * al, customlog * log) break; /*case LFT_REQUEST_SIZE_LINE: */ - /*case LFT_REQUEST_SIZE_HEADERS: */ + case LFT_REQUEST_SIZE_HEADERS: + outoff = al->cache.requestHeadersSize; + dooff =1; + break; /*case LFT_REQUEST_SIZE_BODY: */ /*case LFT_REQUEST_SIZE_BODY_NO_TE: */ @@ -804,7 +1096,9 @@ accessLogCustom(AccessLogEntry * al, customlog * log) break; /*case LFT_REPLY_SIZE_LINE: */ - /*case LFT_REPLY_SIZE_HEADERS: */ + case LFT_REPLY_SIZE_HEADERS: + outint = al->cache.replyHeadersSize; + doint = 1; /*case LFT_REPLY_SIZE_BODY: */ /*case LFT_REPLY_SIZE_BODY_NO_TE: */ @@ -1022,6 +1316,15 @@ accessLogGetNewLogFormatToken(logformat_token * lt, char *def, enum log_quote *q cur++; } + // For upward compatibility, assume "http::" prefix as default prefix + // for all log access formating codes, except those starting + // from "icap::", "adapt::" and "%" + if (strncmp(cur,"http::", 6) == 0 && + strncmp(cur+6, "icap::", 6) != 0 && + strncmp(cur+6, "adapt::", 12) != 0 && *(cur+6) != '%' ) { + cur += 6; + } + lt->type = LFT_NONE; for (lte = logformat_token_table; lte->config != NULL; lte++) { @@ -1046,6 +1349,14 @@ done: switch (lt->type) { +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + + case LFT_ICAP_REQ_HEADER: + + case LFT_ICAP_REP_HEADER: +#endif + case LFT_REQUEST_HEADER: case LFT_REPLY_HEADER: @@ -1064,16 +1375,30 @@ done: lt->data.header.element = cp; - lt->type = (lt->type == LFT_REQUEST_HEADER) ? - LFT_REQUEST_HEADER_ELEM : - LFT_REPLY_HEADER_ELEM; + switch(lt->type) { + case LFT_REQUEST_HEADER: lt->type = LFT_REQUEST_HEADER_ELEM; break; + case LFT_REPLY_HEADER: lt->type = LFT_REPLY_HEADER_ELEM; break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: lt->type = LFT_ICAP_LAST_MATCHED_HEADER_ELEM; break; + case LFT_ICAP_REQ_HEADER: lt->type = LFT_ICAP_REQ_HEADER_ELEM; break; + case LFT_ICAP_REP_HEADER: lt->type = LFT_ICAP_REP_HEADER_ELEM; break; +#endif + default:break; + } } lt->data.header.header = header; } else { - lt->type = (lt->type == LFT_REQUEST_HEADER) ? - LFT_REQUEST_ALL_HEADERS : - LFT_REPLY_ALL_HEADERS; + switch(lt->type) { + case LFT_REQUEST_HEADER: lt->type = LFT_REQUEST_ALL_HEADERS; break; + case LFT_REPLY_HEADER: lt->type = LFT_REPLY_ALL_HEADERS; break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: lt->type = LFT_ICAP_LAST_MATCHED_ALL_HEADERS; break; + case LFT_ICAP_REQ_HEADER: lt->type = LFT_ICAP_REQ_ALL_HEADERS; break; + case LFT_ICAP_REP_HEADER: lt->type = LFT_ICAP_REP_ALL_HEADERS; break; +#endif + default:break; + } Config.onoff.log_mime_hdrs = 1; } @@ -1099,6 +1424,10 @@ done: break; + case LFT_HTTP_SENT_STATUS_CODE_OLD_30: + debugs(46, 0, "WARNING: the \"Hs\" formating code is deprecated use the \">Hs\" instead"); + lt->type = LFT_HTTP_SENT_STATUS_CODE; + break; default: break; } @@ -1160,7 +1489,11 @@ accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definit case LFT_STRING: break; - +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: + case LFT_ICAP_REQ_HEADER_ELEM: + case LFT_ICAP_REP_HEADER_ELEM: +#endif case LFT_REQUEST_HEADER_ELEM: case LFT_REPLY_HEADER_ELEM: @@ -1172,9 +1505,16 @@ accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definit arg = argbuf; - type = (type == LFT_REQUEST_HEADER_ELEM) ? - LFT_REQUEST_HEADER : - LFT_REPLY_HEADER; + switch(type) { + case LFT_REQUEST_HEADER_ELEM: type = LFT_REQUEST_HEADER_ELEM; break; + case LFT_REPLY_HEADER_ELEM: type = LFT_REPLY_HEADER_ELEM; break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: type = LFT_ICAP_LAST_MATCHED_HEADER; break; + case LFT_ICAP_REQ_HEADER_ELEM: type = LFT_ICAP_REQ_HEADER; break; + case LFT_ICAP_REP_HEADER_ELEM: type = LFT_ICAP_REP_HEADER; break; +#endif + default:break; + } break; @@ -1182,9 +1522,22 @@ accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definit case LFT_REPLY_ALL_HEADERS: - type = (type == LFT_REQUEST_ALL_HEADERS) ? - LFT_REQUEST_HEADER : - LFT_REPLY_HEADER; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: + case LFT_ICAP_REQ_ALL_HEADERS: + case LFT_ICAP_REP_ALL_HEADERS: +#endif + + switch(type) { + case LFT_REQUEST_ALL_HEADERS: type = LFT_REQUEST_HEADER; break; + case LFT_REPLY_ALL_HEADERS: type = LFT_REPLY_HEADER; break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: type = LFT_ICAP_LAST_MATCHED_HEADER; break; + case LFT_ICAP_REQ_ALL_HEADERS: type = LFT_ICAP_REQ_HEADER; break; + case LFT_ICAP_REP_ALL_HEADERS: type = LFT_ICAP_REP_HEADER; break; +#endif + default:break; + } break; @@ -1235,7 +1588,7 @@ accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definit storeAppendPrintf(entry, "{%s}", arg); for (te = logformat_token_table; te->config != NULL; te++) { - if (te->token_type == t->type) { + if (te->token_type == type) { storeAppendPrintf(entry, "%s", te->config); break; } @@ -1387,13 +1740,61 @@ accessLogCommon(AccessLogEntry * al, Logfile * logfile) } -void -accessLogLog(AccessLogEntry * al, ACLChecklist * checklist) +#if ICAP_CLIENT +static void +accessLogICAPSquid(AccessLogEntry * al, Logfile * logfile) { - customlog *log; + const char *client = NULL; + const char *user = NULL; + char tmp[MAX_IPSTRLEN], clientbuf[MAX_IPSTRLEN]; - if (LogfileStatus != LOG_ENABLE) - return; + if (al->cache.caddr.IsAnyAddr()) { // ICAP OPTIONS xactions lack client + client = "-"; + } else { + if (Config.onoff.log_fqdn) + client = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); + if (!client) + client = al->cache.caddr.NtoA(clientbuf, MAX_IPSTRLEN); + } + + user = accessLogFormatName(al->cache.authuser); + + if (!user) + user = accessLogFormatName(al->cache.extuser); + +#if USE_SSL + + if (!user) + user = accessLogFormatName(al->cache.ssluser); + +#endif + + if (!user) + user = accessLogFormatName(al->cache.rfc931); + + if (user && !*user) + safe_free(user); + + logfilePrintf(logfile, "%9ld.%03d %6d %s -/%03d %"PRId64" %s %s %s -/%s -\n", + (long int) current_time.tv_sec, + (int) current_time.tv_usec / 1000, + + al->icap.trTime, + client, + + al->icap.resStatus, + al->icap.bytesRead, + Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod), + al->icap.reqUri.termedBuf(), + user ? user : dash_str, + al->icap.hostAddr.NtoA(tmp, MAX_IPSTRLEN)); + safe_free(user); +} +#endif + +void +accessLogLogTo(customlog* log, AccessLogEntry * al, ACLChecklist * checklist) +{ if (al->url == NULL) al->url = dash_str; @@ -1411,7 +1812,7 @@ accessLogLog(AccessLogEntry * al, ACLChecklist * checklist) if (al->hier.host[0] == '\0') xstrncpy(al->hier.host, dash_str, SQUIDHOSTNAMELEN); - for (log = Config.Log.accesslogs; log; log = log->next) { + for (; log; log = log->next) { if (checklist && log->aclList && !checklist->matchAclListFast(log->aclList)) continue; @@ -1441,6 +1842,13 @@ accessLogLog(AccessLogEntry * al, ACLChecklist * checklist) break; +#if ICAP_CLIENT + case CLF_ICAP_SQUID: + accessLogICAPSquid(al, log->logfile); + + break; +#endif + case CLF_NONE: goto last; @@ -1458,7 +1866,15 @@ accessLogLog(AccessLogEntry * al, ACLChecklist * checklist) last: (void)0; /* NULL statement for label */ +} +void +accessLogLog(AccessLogEntry * al, ACLChecklist * checklist) +{ + if (LogfileStatus != LOG_ENABLE) + return; + + accessLogLogTo(Config.Log.accesslogs, al, checklist); #if MULTICAST_MISS_STREAM if (al->cache.code != LOG_TCP_MISS) @@ -1534,7 +1950,10 @@ HierarchyLogEntry::HierarchyLogEntry() : code(HIER_NONE), cd_lookup(LOOKUP_NONE), n_choices(0), - n_ichoices(0) + n_ichoices(0), + peer_reply_status(HTTP_STATUS_NONE), + peer_response_time(-1), + total_response_time(-1) { memset(host, '\0', SQUIDHOSTNAMELEN); memset(cd_host, '\0', SQUIDHOSTNAMELEN); @@ -1544,6 +1963,12 @@ HierarchyLogEntry::HierarchyLogEntry() : store_complete_stop.tv_sec =0; store_complete_stop.tv_usec =0; + + peer_http_request_sent.tv_sec = 0; + peer_http_request_sent.tv_usec = 0; + + first_conn_start.tv_sec = 0; + first_conn_start.tv_usec = 0; } void @@ -1580,6 +2005,28 @@ accessLogInit(void) log->logfile = logfileOpen(log->filename, MAX_URL << 1, 1); LogfileStatus = LOG_ENABLE; + +#if USE_ADAPTATION || ICAP_CLIENT + alLogformatHasAdaptToken = false; + alLogformatHasIcapToken = false; + for (logformat_token * curr_token = (log->logFormat?log->logFormat->format:NULL); curr_token; curr_token = curr_token->next) + { +#if USE_ADAPTATION + if (curr_token->type == LTF_ADAPTATION_SUM_XACT_TIMES || + curr_token->type == LTF_ADAPTATION_ALL_XACT_TIMES) { + alLogformatHasAdaptToken = true; + } +#endif +#if ICAP_CLIENT + if (curr_token->type == LFT_ICAP_LAST_MATCHED_HEADER || + curr_token->type == LFT_ICAP_LAST_MATCHED_HEADER_ELEM || + curr_token->type == LFT_ICAP_LAST_MATCHED_ALL_HEADERS) + { + alLogformatHasIcapToken = true; + } +#endif + } +#endif } #if HEADERS_LOG @@ -1848,11 +2295,20 @@ void accessLogFreeMemory(AccessLogEntry * aLogEntry) { safe_free(aLogEntry->headers.request); + +#if ICAP_CLIENT + safe_free(aLogEntry->headers.icap); +#endif + safe_free(aLogEntry->headers.reply); safe_free(aLogEntry->cache.authuser); HTTPMSGUNLOCK(aLogEntry->reply); HTTPMSGUNLOCK(aLogEntry->request); +#if ICAP_CLIENT + HTTPMSGUNLOCK(aLogEntry->icap.reply); + HTTPMSGUNLOCK(aLogEntry->icap.request); +#endif } int diff --git a/src/acl/DestinationDomain.cc b/src/acl/DestinationDomain.cc index 96be7b7e82..4e674300e8 100644 --- a/src/acl/DestinationDomain.cc +++ b/src/acl/DestinationDomain.cc @@ -58,14 +58,15 @@ DestinationDomainLookup::checkForAsync(ACLChecklist *cl) const } void -DestinationDomainLookup::LookupDone(const char *fqdn, void *data) +DestinationDomainLookup::LookupDone(const char *fqdn, const DnsLookupDetails &details, void *data) { - ACLChecklist *checklist = (ACLChecklist *)data; + ACLFilledChecklist *checklist = Filled((ACLChecklist*)data); assert (checklist->asyncState() == DestinationDomainLookup::Instance()); checklist->asyncInProgress(false); checklist->changeState (ACLChecklist::NullState::Instance()); - Filled(checklist)->markDestinationDomainChecked(); + checklist->markDestinationDomainChecked(); + checklist->request->recordLookup(details); checklist->check(); } diff --git a/src/acl/DestinationDomain.h b/src/acl/DestinationDomain.h index bf90c5df38..3245fdea56 100644 --- a/src/acl/DestinationDomain.h +++ b/src/acl/DestinationDomain.h @@ -73,7 +73,7 @@ public: private: static DestinationDomainLookup instance_; - static void LookupDone(const char *, void *); + static void LookupDone(const char *, const DnsLookupDetails &, void *); }; /// \ingroup ACLAPI diff --git a/src/acl/DestinationIp.cc b/src/acl/DestinationIp.cc index 96693d3a4f..39d3bf9fc6 100644 --- a/src/acl/DestinationIp.cc +++ b/src/acl/DestinationIp.cc @@ -86,11 +86,12 @@ DestinationIPLookup::checkForAsync(ACLChecklist *cl)const } void -DestinationIPLookup::LookupDone(const ipcache_addrs * ia, void *data) +DestinationIPLookup::LookupDone(const ipcache_addrs *, const DnsLookupDetails &details, void *data) { - ACLChecklist *checklist = (ACLChecklist *)data; + ACLFilledChecklist *checklist = Filled((ACLChecklist*)data); assert (checklist->asyncState() == DestinationIPLookup::Instance()); - Filled(checklist)->request->flags.destinationIPLookupCompleted(); + checklist->request->flags.destinationIPLookupCompleted(); + checklist->request->recordLookup(details); checklist->asyncInProgress(false); checklist->changeState (ACLChecklist::NullState::Instance()); checklist->check(); diff --git a/src/acl/SourceDomain.cc b/src/acl/SourceDomain.cc index e49ef7a47f..44e71c6c63 100644 --- a/src/acl/SourceDomain.cc +++ b/src/acl/SourceDomain.cc @@ -39,6 +39,7 @@ #include "acl/Checklist.h" #include "acl/RegexData.h" #include "acl/DomainData.h" +#include "HttpRequest.h" SourceDomainLookup SourceDomainLookup::instance_; @@ -56,14 +57,15 @@ SourceDomainLookup::checkForAsync(ACLChecklist *checklist) const } void -SourceDomainLookup::LookupDone(const char *fqdn, void *data) +SourceDomainLookup::LookupDone(const char *fqdn, const DnsLookupDetails &details, void *data) { - ACLChecklist *checklist = (ACLChecklist *)data; + ACLFilledChecklist *checklist = Filled((ACLChecklist*)data); assert (checklist->asyncState() == SourceDomainLookup::Instance()); checklist->asyncInProgress(false); checklist->changeState (ACLChecklist::NullState::Instance()); - Filled(checklist)->markSourceDomainChecked(); + checklist->markSourceDomainChecked(); + checklist->request->recordLookup(details); checklist->check(); } diff --git a/src/acl/SourceDomain.h b/src/acl/SourceDomain.h index 69629cefdc..21eac7f94c 100644 --- a/src/acl/SourceDomain.h +++ b/src/acl/SourceDomain.h @@ -67,7 +67,7 @@ public: private: static SourceDomainLookup instance_; - static void LookupDone(const char *, void *); + static void LookupDone(const char *, const DnsLookupDetails &, void *); }; class ACLSourceDomain diff --git a/src/adaptation/AccessCheck.cc b/src/adaptation/AccessCheck.cc index 96bc2d9f47..d9df86181a 100644 --- a/src/adaptation/AccessCheck.cc +++ b/src/adaptation/AccessCheck.cc @@ -53,11 +53,22 @@ Adaptation::AccessCheck::AccessCheck(Method aMethod, acl_checklist = NULL; +#if ICAP_CLIENT + Adaptation::Icap::History::Pointer h = req->icapHistory(); + if (h != NULL) + h->start("ACL"); +#endif + debugs(93, 5, HERE << "AccessCheck constructed for " << methodStr(method) << " " << vectPointStr(point)); } Adaptation::AccessCheck::~AccessCheck() { +#if ICAP_CLIENT + Adaptation::Icap::History::Pointer h = req->icapHistory(); + if (h != NULL) + h->stop("ACL"); +#endif HTTPMSGUNLOCK(req); HTTPMSGUNLOCK(rep); if (callback_data) diff --git a/src/adaptation/Config.cc b/src/adaptation/Config.cc index 0b2ca27e5e..e2553a677e 100644 --- a/src/adaptation/Config.cc +++ b/src/adaptation/Config.cc @@ -41,9 +41,11 @@ #include "adaptation/Service.h" #include "adaptation/AccessRule.h" #include "adaptation/ServiceGroups.h" +#include "adaptation/History.h" bool Adaptation::Config::Enabled = false; +char *Adaptation::Config::masterx_shared_name = NULL; void Adaptation::Config::parseService() @@ -117,6 +119,7 @@ Adaptation::Config::Finalize(bool enabled) Enabled = enabled; debugs(93,1, "Adaptation support is " << (Enabled ? "on" : "off.")); + History::Configure(); FinalizeEach(AllServices(), "message adaptation services"); FinalizeEach(AllGroups(), "message adaptation service groups"); FinalizeEach(AllRules(), "message adaptation access rules"); diff --git a/src/adaptation/Config.h b/src/adaptation/Config.h index 66603e7750..ffe97d1184 100644 --- a/src/adaptation/Config.h +++ b/src/adaptation/Config.h @@ -41,6 +41,10 @@ public: public: static bool Enabled; // true if at least one adaptation mechanism is + // these are global squid.conf options, documented elsewhere + static char *masterx_shared_name; // global TODO: do we need TheConfig? + // Options below are accessed via Icap::TheConfig or Ecap::TheConfig + // TODO: move ICAP-specific options to Icap::Config and add TheConfig int onoff; int send_client_ip; int send_client_username; diff --git a/src/adaptation/History.cc b/src/adaptation/History.cc new file mode 100644 index 0000000000..5d98d912a8 --- /dev/null +++ b/src/adaptation/History.cc @@ -0,0 +1,127 @@ +#include "config.h" +#include "globals.h" +#include "TextException.h" +#include "SquidTime.h" +#include "HttpRequest.h" /* for alLogformatHasAdaptToken */ +#include "adaptation/Config.h" +#include "adaptation/History.h" + +Adaptation::History::Entry::Entry(const String &sid, const timeval &when): + service(sid), start(when), theRptm(-1), retried(false) +{ +} + +Adaptation::History::Entry::Entry(): + start(current_time), theRptm(-1), retried(false) +{ +} + +void Adaptation::History::Entry::stop() +{ + // theRptm may already be set if the access log entry has already been made + (void)rptm(); // will cache result in theRptm if not set already +} + +int Adaptation::History::Entry::rptm() +{ + if (theRptm < 0) + theRptm = tvSubMsec(start, current_time); + return theRptm; +} + + +int Adaptation::History::recordXactStart(const String &sid, const timeval &when, bool retrying) +{ + if (retrying) { + Must(!theEntries.empty()); // or there would be nothing to retry + theEntries.back().retried = true; + } + theEntries.push_back(Adaptation::History::Entry(sid, when)); + return theEntries.size() - 1; // record position becomes history ID +} + +void Adaptation::History::recordXactFinish(int hid) +{ + Must(0 <= hid && hid < static_cast(theEntries.size())); + theEntries[hid].stop(); +} + +void Adaptation::History::allLogString(const char *serviceId, String &s) +{ + s=""; + bool prevWasRetried = false; + // XXX: Fix Vector<> so that we can use const_iterator here + typedef Adaptation::History::Entries::iterator ECI; + for (ECI i = theEntries.begin(); i != theEntries.end(); ++i) { + // TODO: here and below, optimize service ID comparison? + if (!serviceId || i->service == serviceId) { + if (s.size() > 0) // not the first logged time, must delimit + s.append(prevWasRetried ? "+" : ","); + + char buf[64]; + snprintf(buf, sizeof(buf), "%d", i->rptm()); + s.append(buf); + + // continue; we may have two identical services (e.g., for retries) + } + prevWasRetried = i->retried; + } +} + +void Adaptation::History::sumLogString(const char *serviceId, String &s) +{ + s=""; + int retriedRptm = 0; // sum of rptm times of retried transactions + typedef Adaptation::History::Entries::iterator ECI; + for (ECI i = theEntries.begin(); i != theEntries.end(); ++i) { + if (i->retried) { // do not log retried xact but accumulate their time + retriedRptm += i->rptm(); + } else + if (!serviceId || i->service == serviceId) { + if (s.size() > 0) // not the first logged time, must delimit + s.append(","); + + char buf[64]; + snprintf(buf, sizeof(buf), "%d", retriedRptm + i->rptm()); + s.append(buf); + + // continue; we may have two identical services (e.g., for retries) + } + + if (!i->retried) + retriedRptm = 0; + } + + // the last transaction is never retried or it would not be the last + Must(!retriedRptm); +} + +void Adaptation::History::updateXxRecord(const char *name, const String &value) +{ + theXxName = name; + theXxValue = value; +} + +bool Adaptation::History::getXxRecord(String &name, String &value) const +{ + if (theXxName.size() <= 0) + return false; + + name = theXxName; + value = theXxValue; + return true; +} + + +bool Adaptation::History::Enabled = false; + +void Adaptation::History::Configure() +{ + const bool loggingNeedsUs = LogfileStatus == LOG_ENABLE && + alLogformatHasAdaptToken; + + Enabled = Adaptation::Config::Enabled && + (loggingNeedsUs || Adaptation::Config::masterx_shared_name); + + // TODO: should we disable unneeded _parts_ of the history? +} diff --git a/src/adaptation/History.h b/src/adaptation/History.h new file mode 100644 index 0000000000..203835c303 --- /dev/null +++ b/src/adaptation/History.h @@ -0,0 +1,67 @@ +#ifndef SQUID_ADAPT_HISTORY_H +#define SQUID_ADAPT_HISTORY_H + +#include "RefCount.h" +#include "Array.h" +#include "SquidString.h" + +namespace Adaptation { + + +/// collects information about adaptations related to a master transaction +class History: public RefCountable { +public: + typedef RefCount Pointer; + + /// record the start of a xact, return xact history ID + int recordXactStart(const String &serviceId, const timeval &when, bool retrying); + + /// record the end of a xact identified by its history ID + void recordXactFinish(int hid); + + /// dump individual xaction times to a string + void allLogString(const char *serviceId, String &buf); + + /// dump xaction times, merging retried and retry times together + void sumLogString(const char *serviceId, String &buf); + + /// sets or resets a cross-transactional database record + void updateXxRecord(const char *name, const String &value); + + /// returns true and fills the record fields iff there is a db record + bool getXxRecord(String &name, String &value) const; + + static bool Enabled; ///< whether some configuration options require it + static void Configure(); ///< determines whether the history is needed + +private: + /// single Xaction stats (i.e., a historical record entry) + class Entry { + public: + Entry(const String &serviceId, const timeval &when); + Entry(); // required by Vector<> + + void stop(); ///< updates stats on transaction end + int rptm(); ///< returns response time [msec], calculates it if needed + + String service; ///< adaptation service ID + timeval start; ///< when the xaction was started + + private: + int theRptm; ///< calculated and cached response time value in msec + + public: + bool retried; ///< whether the xaction was replaced by another + }; + + typedef Vector Entries; + Entries theEntries; ///< historical record, in the order of xact starts + + // theXx* will become a map, but we only support one record + String theXxName; ///< name part of the cross-transactional database record + String theXxValue; ///< value part of the cross-xactional database record +}; + +} // namespace Adaptation + +#endif diff --git a/src/adaptation/Initiate.h b/src/adaptation/Initiate.h index 5b4875b283..ac76568e49 100644 --- a/src/adaptation/Initiate.h +++ b/src/adaptation/Initiate.h @@ -32,6 +32,7 @@ public: bool isThere(); // we have a valid initiator pointer Initiator *ptr(); // asserts isThere() + void *theCbdata() { return cbdata;} private: InitiatorHolder &operator =(const InitiatorHolder &anInitiator); @@ -74,6 +75,10 @@ protected: InitiatorHolder theInitiator; ServicePointer theService; + +private: + Initiate(const Initiate &); // no definition + Initiate &operator =(const Initiate &); // no definition }; } // namespace Adaptation diff --git a/src/adaptation/Makefile.am b/src/adaptation/Makefile.am index 7f7801a9cc..745279be71 100644 --- a/src/adaptation/Makefile.am +++ b/src/adaptation/Makefile.am @@ -36,7 +36,9 @@ libadaptation_la_SOURCES = \ ServiceConfig.cc \ ServiceConfig.h \ ServiceGroups.cc \ - ServiceGroups.h + ServiceGroups.h \ + History.cc \ + History.h # add libraries for specific adaptation schemes libadaptation_la_LIBADD = @ECAP_LIBS@ @ECAPLIB@ @ICAP_LIBS@ diff --git a/src/adaptation/ecap/XactionRep.cc b/src/adaptation/ecap/XactionRep.cc index 2a96ae6e78..62db6b45ea 100644 --- a/src/adaptation/ecap/XactionRep.cc +++ b/src/adaptation/ecap/XactionRep.cc @@ -5,6 +5,7 @@ #include "TextException.h" #include "HttpRequest.h" #include "HttpReply.h" +#include "SquidTime.h" #include "adaptation/ecap/XactionRep.h" CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Ecap::XactionRep, XactionRep); @@ -16,7 +17,9 @@ Adaptation::Ecap::XactionRep::XactionRep(Adaptation::Initiator *anInitiator, AsyncJob("Adaptation::Ecap::XactionRep"), Adaptation::Initiate("Adaptation::Ecap::XactionRep", anInitiator, aService), theVirginRep(virginHeader), theCauseRep(NULL), - proxyingVb(opUndecided), proxyingAb(opUndecided), canAccessVb(false), + proxyingVb(opUndecided), proxyingAb(opUndecided), + adaptHistoryId(-1), + canAccessVb(false), abProductionFinished(false), abProductionAtEnd(false) { if (virginCause) @@ -48,6 +51,15 @@ Adaptation::Ecap::XactionRep::start() else proxyingVb = opNever; + const HttpRequest *request = dynamic_cast (theCauseRep ? + theCauseRep->raw().header : theVirginRep.raw().header); + Must(request); + Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { + // retrying=false because ecap never retries transactions + adaptHistoryId = ah->recordXactStart(service().cfg().key, current_time, false); + } + theMaster->start(); } @@ -74,6 +86,14 @@ Adaptation::Ecap::XactionRep::swanSong() } terminateMaster(); + + const HttpRequest *request = dynamic_cast(theCauseRep ? + theCauseRep->raw().header : theVirginRep.raw().header); + Must(request); + Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL && adaptHistoryId >= 0) + ah->recordXactFinish(adaptHistoryId); + Adaptation::Initiate::swanSong(); } diff --git a/src/adaptation/ecap/XactionRep.h b/src/adaptation/ecap/XactionRep.h index 3ca3017227..476290e9e6 100644 --- a/src/adaptation/ecap/XactionRep.h +++ b/src/adaptation/ecap/XactionRep.h @@ -92,6 +92,7 @@ private: typedef enum { opUndecided, opOn, opComplete, opNever } OperationState; OperationState proxyingVb; // delivering virgin body from core to adapter OperationState proxyingAb; // delivering adapted body from adapter to core + int adaptHistoryId; ///< adaptation history slot reservation bool canAccessVb; // virgin BodyPipe content is accessible bool abProductionFinished; // whether adapter has finished producing ab bool abProductionAtEnd; // whether adapter produced a complete ab diff --git a/src/adaptation/icap/Config.cc b/src/adaptation/icap/Config.cc index fb184e8fa3..e6f6ba8ddd 100644 --- a/src/adaptation/icap/Config.cc +++ b/src/adaptation/icap/Config.cc @@ -47,7 +47,7 @@ Adaptation::Icap::Config Adaptation::Icap::TheConfig; Adaptation::Icap::Config::Config(): preview_enable(0), preview_size(0), connect_timeout_raw(0), io_timeout_raw(0), reuse_connections(0), - client_username_header(NULL), client_username_encode(0) + client_username_header(NULL), client_username_encode(0), repeat(NULL) { } diff --git a/src/adaptation/icap/Config.h b/src/adaptation/icap/Config.h index 9f5dd31531..e338d5139d 100644 --- a/src/adaptation/icap/Config.h +++ b/src/adaptation/icap/Config.h @@ -41,13 +41,12 @@ #include "adaptation/Config.h" #include "adaptation/icap/ServiceRep.h" +class acl_access; namespace Adaptation { namespace Icap { -class acl_access; - class ConfigParser; class Config: public Adaptation::Config @@ -62,6 +61,8 @@ public: int reuse_connections; char* client_username_header; int client_username_encode; + acl_access *repeat; ///< icap_retry ACL in squid.conf + int repeat_limit; ///< icap_retry_limit in squid.conf Config(); ~Config(); diff --git a/src/adaptation/icap/Elements.cc b/src/adaptation/icap/Elements.cc index fa269eef1c..5d1f084732 100644 --- a/src/adaptation/icap/Elements.cc +++ b/src/adaptation/icap/Elements.cc @@ -2,3 +2,15 @@ #include "adaptation/icap/Elements.h" // TODO: remove this file? +namespace Adaptation { +namespace Icap { + +const XactOutcome xoUnknown = "ICAP_ERR_UNKNOWN"; +const XactOutcome xoError = "ICAP_ERR_OTHER"; +const XactOutcome xoOpt = "ICAP_OPT"; +const XactOutcome xoEcho = "ICAP_ECHO"; +const XactOutcome xoModified = "ICAP_MOD"; +const XactOutcome xoSatisfied = "ICAP_SAT"; + +} // namespace Icap +} // namespace Adaptation diff --git a/src/adaptation/icap/Elements.h b/src/adaptation/icap/Elements.h index e84723a7b4..ba5af8c866 100644 --- a/src/adaptation/icap/Elements.h +++ b/src/adaptation/icap/Elements.h @@ -43,6 +43,7 @@ namespace Adaptation { namespace Icap { +//TODO: remove the ICAP namespace namespace ICAP { using Adaptation::Method; using Adaptation::methodNone; @@ -59,6 +60,13 @@ using Adaptation::methodStr; using Adaptation::vectPointStr; } +typedef const char *XactOutcome; ///< transaction result for logging +extern const XactOutcome xoUnknown; ///< initial value: outcome was not set +extern const XactOutcome xoError; ///< all kinds of transaction errors +extern const XactOutcome xoOpt; ///< OPTION transaction +extern const XactOutcome xoEcho; ///< preserved virgin message (ICAP 204) +extern const XactOutcome xoModified; ///< replaced virgin msg with adapted +extern const XactOutcome xoSatisfied; ///< request satisfaction } // namespace Icap } // namespace Adaptation diff --git a/src/adaptation/icap/History.cc b/src/adaptation/icap/History.cc new file mode 100644 index 0000000000..56e5290827 --- /dev/null +++ b/src/adaptation/icap/History.cc @@ -0,0 +1,103 @@ +#include "adaptation/icap/History.h" +#include "squid.h" +#include "globals.h" +#include "SquidTime.h" + +Adaptation::Icap::History::History(): mergeOfIcapHeaders(hoRequest), + lastIcapHeader(hoRequest), logType(LOG_TAG_NONE), req_sz(0), + pastTime(0), concurrencyLevel(0) +{ +} + +Adaptation::Icap::History::History(const Adaptation::Icap::History& ih) +{ + assign(ih); +} + +Adaptation::Icap::History::~History() +{ + mergeOfIcapHeaders.clean(); + lastIcapHeader.clean(); + rfc931.clean(); +#if USE_SSL + ssluser.clean(); +#endif + log_uri.clean(); +} + +void Adaptation::Icap::History::assign(const Adaptation::Icap::History& ih) +{ + mergeOfIcapHeaders.clean(); + mergeOfIcapHeaders.update(&ih.mergeOfIcapHeaders, NULL); + lastIcapHeader.clean(); + lastIcapHeader.update(&ih.lastIcapHeader, NULL); + rfc931 = ih.rfc931; + +#if USE_SSL + ssluser = ih.ssluser; +#endif + + logType = ih.logType; + log_uri = ih.log_uri; + req_sz = ih.req_sz; + pastTime = ih.pastTime; + currentStart = ih.currentStart; + concurrencyLevel = ih.concurrencyLevel; + debugs(93,7, HERE << this << " = " << &ih); +} + +Adaptation::Icap::History& Adaptation::Icap::History::operator=(const History& ih) +{ + if (this != &ih) + assign(ih); + return *this; +} + +void Adaptation::Icap::History::setIcapLastHeader(const HttpHeader * lih) +{ + lastIcapHeader.clean(); + lastIcapHeader.update(lih, NULL); +} + +void Adaptation::Icap::History::mergeIcapHeaders(const HttpHeader * lih) +{ + mergeOfIcapHeaders.update(lih, NULL); + mergeOfIcapHeaders.compact(); +} + +void Adaptation::Icap::History::start(const char *context) +{ + if (!concurrencyLevel++) + currentStart = current_time; + + debugs(93,4, HERE << "start " << context << " level=" << concurrencyLevel + << " time=" << pastTime << ' ' << this); +} + +void Adaptation::Icap::History::stop(const char *context) +{ + if (!concurrencyLevel) { + debugs(93,1, HERE << "Internal error: poor history accounting " << this); + return; + } + + const int current = currentTime(); + debugs(93,4, HERE << "stop " << context << " level=" << concurrencyLevel << + " time=" << pastTime << '+' << current << ' ' << this); + + if (!--concurrencyLevel) + pastTime += current; +} + +int Adaptation::Icap::History::processingTime() const +{ + const int total = pastTime + currentTime(); + debugs(93,7, HERE << " current total: " << total << ' ' << this); + return total; +} + +int Adaptation::Icap::History::currentTime() const +{ + return concurrencyLevel > 0 ? + max(0, tvSubMsec(currentStart, current_time)) : 0; +} diff --git a/src/adaptation/icap/History.h b/src/adaptation/icap/History.h new file mode 100644 index 0000000000..f3f398ee5a --- /dev/null +++ b/src/adaptation/icap/History.h @@ -0,0 +1,60 @@ +#ifndef SQUID_ICAPHISTORY_H +#define SQUID_ICAPHISTORY_H + +#include "RefCount.h" +#include "HttpHeader.h" +#include "enums.h" + +namespace Adaptation { +namespace Icap { + +/// collects information about ICAP processing related to an HTTP transaction +class History: public RefCountable { +public: + typedef RefCount Pointer; + + History(); + History(const History& ih); + + ~History(); + + History& operator=(const History& ih); + + ///store the last reply header from ICAP server + void setIcapLastHeader(const HttpHeader * lih); + ///merge new header and stored one + void mergeIcapHeaders(const HttpHeader * lih); + + /// record the start of an ICAP processing interval + void start(const char *context); + /// note the end of an ICAP processing interval + void stop(const char *context); + + /// returns the total time of all ICAP processing intervals + int processingTime() const; + + HttpHeader mergeOfIcapHeaders; ///< Merge of REQMOD and RESPMOD responses. If both responses contain the header, the one from the last response will be used + HttpHeader lastIcapHeader; ///< Last received reply from ICAP server + String rfc931; ///< the username from ident +#if USE_SSL + String ssluser; ///< the username from SSL +#endif + log_type logType; ///< the squid request status (TCP_MISS etc) + + String log_uri; ///< the request uri + size_t req_sz; ///< the request size + +private: + void assign(const History &); + + int currentTime() const; ///< time since current start or zero + + timeval currentStart; ///< when the current processing interval started + int pastTime; ///< sum of closed processing interval durations + int concurrencyLevel; ///< number of concurrent processing threads +}; + +} // namespace Icap +} // namespace Adaptation + +#endif /*SQUID_HISTORY_H*/ diff --git a/src/adaptation/icap/Launcher.cc b/src/adaptation/icap/Launcher.cc index 1c5c730942..4b64e45f69 100644 --- a/src/adaptation/icap/Launcher.cc +++ b/src/adaptation/icap/Launcher.cc @@ -4,10 +4,14 @@ #include "squid.h" #include "TextException.h" +#include "HttpRequest.h" +#include "HttpReply.h" +#include "acl/FilledChecklist.h" #include "HttpMsg.h" #include "adaptation/icap/Launcher.h" #include "adaptation/icap/Xaction.h" #include "adaptation/icap/ServiceRep.h" +#include "adaptation/icap/Config.h" Adaptation::Icap::Launcher::Launcher(const char *aTypeName, @@ -28,17 +32,20 @@ void Adaptation::Icap::Launcher::start() Adaptation::Initiate::start(); Must(theInitiator); - launchXaction(false); + launchXaction("first"); } -void Adaptation::Icap::Launcher::launchXaction(bool final) +void Adaptation::Icap::Launcher::launchXaction(const char *xkind) { Must(!theXaction); ++theLaunches; - debugs(93,4, HERE << "launching xaction #" << theLaunches); + debugs(93,4, HERE << "launching " << xkind << " xaction #" << theLaunches); Adaptation::Icap::Xaction *x = createXaction(); - if (final) + x->attempts = theLaunches; + if (theLaunches > 1) x->disableRetries(); + if (theLaunches >= TheConfig.repeat_limit) + x->disableRepeats("over icap_retry_limit"); theXaction = initiateAdaptation(x); Must(theXaction); } @@ -60,19 +67,33 @@ void Adaptation::Icap::Launcher::noteInitiatorAborted() } +// XXX: this call is unused by ICAPXaction in favor of ICAPLauncher::noteXactAbort void Adaptation::Icap::Launcher::noteAdaptationQueryAbort(bool final) { + debugs(93,5, HERE << "launches: " << theLaunches << "; final: " << final); clearAdaptation(theXaction); + + Must(done()); // swanSong will notify the initiator +} + +void Adaptation::Icap::Launcher::noteXactAbort(XactAbortInfo &info) +{ + debugs(93,5, HERE << "theXaction:" << theXaction << " launches: " << theLaunches); - // TODO: add more checks from FwdState::checkRetry()? - if (!final && theLaunches < 2 && !shutting_down) { - launchXaction(true); + // TODO: add more checks from FwdState::checkRetry()? + if (canRetry(info)) { + clearAdaptation(theXaction); + launchXaction("retry"); + } + else if (canRepeat(info)) { + clearAdaptation(theXaction); + launchXaction("repeat"); } else { - debugs(93,3, HERE << "cannot retry the failed ICAP xaction; tries: " << - theLaunches << "; final: " << final); - Must(done()); // swanSong will notify the initiator - } + debugs(93,3, HERE << "cannot retry or repeat a failed transaction"); + clearAdaptation(theXaction); + Must(done()); // swanSong will notify the initiator + } } bool Adaptation::Icap::Launcher::doneAll() const @@ -90,3 +111,60 @@ void Adaptation::Icap::Launcher::swanSong() Adaptation::Initiate::swanSong(); } + +bool Adaptation::Icap::Launcher::canRetry(Adaptation::Icap::XactAbortInfo &info) const +{ + // We do not check and can exceed zero repeat limit when retrying. + // This is by design as the limit does not apply to pconn retrying. + return !shutting_down && info.isRetriable; +} + +bool Adaptation::Icap::Launcher::canRepeat(Adaptation::Icap::XactAbortInfo &info) const +{ + debugs(93,9, HERE << shutting_down); + if (theLaunches >= TheConfig.repeat_limit || shutting_down) + return false; + + debugs(93,9, HERE << info.isRepeatable); // TODO: update and use status() + if (!info.isRepeatable) + return false; + + debugs(93,9, HERE << info.icapReply); + if (!info.icapReply) // did not get to read an ICAP reply; a timeout? + return true; + + debugs(93,9, HERE << info.icapReply->sline.status); + if (!info.icapReply->sline.status) // failed to parse the reply; I/O err + return true; + + ACLFilledChecklist *cl = + new ACLFilledChecklist(TheConfig.repeat, info.icapRequest, dash_str); + cl->reply = HTTPMSGLOCK(info.icapReply); + + const bool result = cl->fastCheck(); + delete cl; + return result; +} + +/* ICAPXactAbortInfo */ + +Adaptation::Icap::XactAbortInfo::XactAbortInfo(HttpRequest *anIcapRequest, + HttpReply *anIcapReply, bool beRetriable, bool beRepeatable): + icapRequest(anIcapRequest ? HTTPMSGLOCK(anIcapRequest) : NULL), + icapReply(anIcapReply ? HTTPMSGLOCK(anIcapReply) : NULL), + isRetriable(beRetriable), isRepeatable(beRepeatable) +{ +} + +Adaptation::Icap::XactAbortInfo::XactAbortInfo(const Adaptation::Icap::XactAbortInfo &i): + icapRequest(i.icapRequest ? HTTPMSGLOCK(i.icapRequest) : NULL), + icapReply(i.icapReply ? HTTPMSGLOCK(i.icapReply) : NULL), + isRetriable(i.isRetriable), isRepeatable(i.isRepeatable) +{ +} + +Adaptation::Icap::XactAbortInfo::~XactAbortInfo() +{ + HTTPMSGUNLOCK(icapRequest); + HTTPMSGUNLOCK(icapReply); +} diff --git a/src/adaptation/icap/Launcher.h b/src/adaptation/icap/Launcher.h index 7c76b96ee0..cef30e86f7 100644 --- a/src/adaptation/icap/Launcher.h +++ b/src/adaptation/icap/Launcher.h @@ -64,6 +64,7 @@ namespace Adaptation namespace Icap { class Xaction; +class XactAbortInfo; // Note: Initiate must be the first parent for cbdata to work. We use // a temporary InitaitorHolder/toCbdata hacks and do not call cbdata @@ -79,6 +80,11 @@ public: // Adaptation::Initiator: asynchronous communication with the current transaction virtual void noteAdaptationAnswer(HttpMsg *message); + virtual void noteXactAbort(XactAbortInfo &info); + +private: + bool canRetry(XactAbortInfo &info) const; //< true if can retry in the case of persistent connection failures + bool canRepeat(XactAbortInfo &info) const; //< true if can repeat in the case of no or unsatisfactory response virtual void noteAdaptationQueryAbort(bool final); protected: @@ -90,14 +96,57 @@ protected: // creates the right ICAP transaction using stored configuration params virtual Xaction *createXaction() = 0; - void launchXaction(bool final); + void launchXaction(const char *xkind); - Adaptation::Initiate *theXaction; // current ICAP transaction + Adaptation::Initiate *theXaction; ///< current ICAP transaction int theLaunches; // the number of transaction launches }; +/// helper class to pass information about aborted ICAP requests to +/// the Adaptation::Icap::Launcher class +class XactAbortInfo { +public: + XactAbortInfo(HttpRequest *anIcapRequest, HttpReply *anIcapReply, + bool beRetriable, bool beRepeatable); + XactAbortInfo(const XactAbortInfo &); + ~XactAbortInfo(); + + HttpRequest *icapRequest; + HttpReply *icapReply; + bool isRetriable; + bool isRepeatable; + +private: + XactAbortInfo &operator =(const XactAbortInfo &); // undefined +}; + +/* required by UnaryMemFunT */ +inline std::ostream &operator << (std::ostream &os, Adaptation::Icap::XactAbortInfo info) +{ + // Nothing, it is unused + return os; +} + +/// A Dialer class used to schedule the Adaptation::Icap::Launcher::noteXactAbort call +class XactAbortCall: public UnaryMemFunT { +public: + typedef void (Adaptation::Icap::Launcher::*DialMethod)(Adaptation::Icap::XactAbortInfo &); + XactAbortCall(Adaptation::Icap::Launcher *launcer, DialMethod aMethod, + const Adaptation::Icap::XactAbortInfo &info): + UnaryMemFunT(launcer, NULL, info), + dialMethod(aMethod) + {} + virtual void print(std::ostream &os) const { os << '(' << "retriable:" << arg1.isRetriable << ", repeatable:" << arg1.isRepeatable << ')'; } + +public: + DialMethod dialMethod; + +protected: + virtual void doDial() { (object->*dialMethod)(arg1); } +}; } // namespace Icap } // namespace Adaptation + #endif /* SQUID_ICAPLAUNCHER_H */ diff --git a/src/adaptation/icap/Makefile.am b/src/adaptation/icap/Makefile.am index dbaa577fb5..8be660d41e 100644 --- a/src/adaptation/icap/Makefile.am +++ b/src/adaptation/icap/Makefile.am @@ -22,4 +22,8 @@ libicap_la_SOURCES = \ Xaction.cc \ Xaction.h \ ModXact.cc \ - ModXact.h + ModXact.h \ + icap_log.cc \ + icap_log.h \ + History.cc \ + History.h diff --git a/src/adaptation/icap/ModXact.cc b/src/adaptation/icap/ModXact.cc index 1441dd26bb..c738c22526 100644 --- a/src/adaptation/icap/ModXact.cc +++ b/src/adaptation/icap/ModXact.cc @@ -17,6 +17,9 @@ #include "auth/UserRequest.h" #include "adaptation/icap/Config.h" #include "SquidTime.h" +#include "AccessLogEntry.h" +#include "adaptation/icap/History.h" +#include "adaptation/History.h" // flow and terminology: // HTTP| --> receive --> encode --> write --> |network @@ -41,10 +44,11 @@ Adaptation::Icap::ModXact::ModXact(Adaptation::Initiator *anInitiator, HttpMsg * HttpRequest *virginCause, Adaptation::Icap::ServiceRep::Pointer &aService): AsyncJob("Adaptation::Icap::ModXact"), Adaptation::Icap::Xaction("Adaptation::Icap::ModXact", anInitiator, aService), - icapReply(NULL), virginConsumed(0), bodyParser(NULL), - canStartBypass(false) // too early + canStartBypass(false), // too early + replyBodySize(0), + adaptHistoryId(-1) { assert(virginHeader); @@ -58,8 +62,8 @@ Adaptation::Icap::ModXact::ModXact(Adaptation::Initiator *anInitiator, HttpMsg * // encoding // nothing to do because we are using temporary buffers - // parsing - icapReply = new HttpReply; + // parsing; TODO: do not set until we parse, see ICAPOptXact + icapReply = HTTPMSGLOCK(new HttpReply); icapReply->protoPrefix = "ICAP/"; // TODO: make an IcapReply class? debugs(93,7, HERE << "initialized." << status()); @@ -70,6 +74,11 @@ void Adaptation::Icap::ModXact::start() { Adaptation::Icap::Xaction::start(); + // reserve an adaptation history slot (attempts are known at this time) + Adaptation::History::Pointer ah = virginRequest().adaptHistory(); + if (ah != NULL) + adaptHistoryId = ah->recordXactStart(service().cfg().key, icap_tr_start, attempts > 1); + estimateVirginBody(); // before virgin disappears! canStartBypass = service().cfg().bypass; @@ -101,6 +110,7 @@ void Adaptation::Icap::ModXact::noteServiceReady() startWriting(); } else { disableRetries(); + disableRepeats("ICAP service is unusable"); throw TexcHere("ICAP service is unusable"); } } @@ -131,6 +141,7 @@ void Adaptation::Icap::ModXact::handleCommConnected() // write headers state.writing = State::writingHeaders; + icap_tio_start = current_time; scheduleWrite(requestBuf); } @@ -304,6 +315,14 @@ void Adaptation::Icap::ModXact::closeChunk(MemBuf &buf) buf.append(ICAP::crlf, 2); // chunk-terminating CRLF } +const HttpRequest &Adaptation::Icap::ModXact::virginRequest() const +{ + const HttpRequest *request = virgin.cause ? + virgin.cause : dynamic_cast(virgin.header); + Must(request); + return *request; +} + // did the activity reached the end of the virgin body? bool Adaptation::Icap::ModXact::virginBodyEndReached(const Adaptation::Icap::VirginBodyAct &act) const { @@ -336,7 +355,8 @@ const char *Adaptation::Icap::ModXact::virginContentData(const Adaptation::Icap: void Adaptation::Icap::ModXact::virginConsume() { - debugs(93, 9, HERE << "consumption guards: " << !virgin.body_pipe << isRetriable); + debugs(93, 9, HERE << "consumption guards: " << !virgin.body_pipe << isRetriable << + isRepeatable << canStartBypass); if (!virgin.body_pipe) return; // nothing to consume @@ -345,11 +365,12 @@ void Adaptation::Icap::ModXact::virginConsume() return; // do not consume if we may have to retry later BodyPipe &bp = *virgin.body_pipe; + const bool wantToPostpone = isRepeatable || canStartBypass; // Why > 2? HttpState does not use the last bytes in the buffer // because delayAwareRead() is arguably broken. See // HttpStateData::maybeReadVirginBody for more details. - if (canStartBypass && bp.buf().spaceSize() > 2) { + if (wantToPostpone && bp.buf().spaceSize() > 2) { // Postponing may increase memory footprint and slow the HTTP side // down. Not postponing may increase the number of ICAP errors // if the ICAP service fails. We may also use "potential" space to @@ -381,6 +402,7 @@ void Adaptation::Icap::ModXact::virginConsume() bp.consume(size); virginConsumed += size; Must(!isRetriable); // or we should not be consuming + disableRepeats("consumed content"); disableBypass("consumed content"); } } @@ -477,6 +499,7 @@ void Adaptation::Icap::ModXact::readMore() void Adaptation::Icap::ModXact::handleCommRead(size_t) { Must(!state.doneParsing()); + icap_tio_finish = current_time; parseMore(); readMore(); } @@ -499,6 +522,7 @@ void Adaptation::Icap::ModXact::echoMore() " bytes"); virginBodySending.progress(size); virginConsume(); + disableRepeats("echoed content"); disableBypass("echoed content"); } @@ -521,8 +545,10 @@ bool Adaptation::Icap::ModXact::doneSending() const // stop (or do not start) sending adapted message body void Adaptation::Icap::ModXact::stopSending(bool nicely) { + debugs(93, 7, HERE << "Enter stop sending "); if (doneSending()) return; + debugs(93, 7, HERE << "Proceed with stop sending "); if (state.sending != State::sendingUndecided) { debugs(93, 7, HERE << "will no longer send" << status()); @@ -587,6 +613,7 @@ void Adaptation::Icap::ModXact::bypassFailure() disableBypass("already started to bypass"); Must(!isRetriable); // or we should not be bypassing + // TODO: should the same be enforced for isRepeatable? Check icap_repeat?? prepEchoing(); @@ -623,8 +650,11 @@ void Adaptation::Icap::ModXact::maybeAllocateHttpMsg() if (gotEncapsulated("res-hdr")) { adapted.setHeader(new HttpReply); + setOutcome(service().cfg().method == ICAP::methodReqmod ? + xoSatisfied : xoModified); } else if (gotEncapsulated("req-hdr")) { adapted.setHeader(new HttpRequest); + setOutcome(xoModified); } else throw TexcHere("Neither res-hdr nor req-hdr in maybeAllocateHttpMsg()"); } @@ -654,6 +684,7 @@ void Adaptation::Icap::ModXact::parseHeaders() // called after parsing all headers or when bypassing an exception void Adaptation::Icap::ModXact::startSending() { + disableRepeats("sent headers"); disableBypass("sent headers"); sendAnswer(adapted.header); @@ -700,6 +731,30 @@ void Adaptation::Icap::ModXact::parseIcapHead() break; } + const HttpRequest *request = dynamic_cast(adapted.header); + if (!request) + request = &virginRequest(); + + // update the cross-transactional database if needed (all status codes!) + if (const char *xxName = Adaptation::Config::masterx_shared_name) { + Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { + const String val = icapReply->header.getByName(xxName); + if (val.size() > 0) // XXX: HttpHeader lacks empty value detection + ah->updateXxRecord(xxName, val); + } + } + + // We need to store received ICAP headers for icapHistory(); + if (h != NULL) { + h->mergeIcapHeaders(&icapReply->header); + h->setIcapLastHeader(&icapReply->header); + } + // handle100Continue() manages state.writing on its own. // Non-100 status means the server needs no postPreview data from us. if (state.writing == State::writingPaused) @@ -762,7 +817,9 @@ void Adaptation::Icap::ModXact::handle204NoContent() // We actually start sending (echoig or not) in startSending. void Adaptation::Icap::ModXact::prepEchoing() { + disableRepeats("preparing to echo content"); disableBypass("preparing to echo content"); + setOutcome(xoEcho); // We want to clone the HTTP message, but we do not want // to copy some non-HTTP state parts that HttpMsg kids carry in them. @@ -913,10 +970,14 @@ void Adaptation::Icap::ModXact::parseBody() debugs(93, 5, HERE << "have " << readBuf.contentSize() << " body bytes after " << "parse; parsed all: " << parsed); + replyBodySize += adapted.body_pipe->buf().contentSize(); // TODO: expose BodyPipe::putSize() to make this check simpler and clearer - if (adapted.body_pipe->buf().contentSize() > 0) // parsed something sometime + // TODO: do we really need this if we disable when sending headers? + if (adapted.body_pipe->buf().contentSize() > 0) { // parsed something sometime + disableRepeats("sent adapted content"); disableBypass("sent adapted content"); + } if (parsed) { stopParsing(); @@ -1014,14 +1075,74 @@ void Adaptation::Icap::ModXact::swanSong() stopWriting(false); stopSending(false); - if (icapReply) { - delete icapReply; - icapReply = NULL; - } + // update adaptation history if start was called and we reserved a slot + Adaptation::History::Pointer ah = virginRequest().adaptHistory(); + if (ah != NULL && adaptHistoryId >= 0) + ah->recordXactFinish(adaptHistoryId); Adaptation::Icap::Xaction::swanSong(); } +void prepareLogWithRequestDetails(HttpRequest *, AccessLogEntry *); + +void Adaptation::Icap::ModXact::finalizeLogInfo() +{ + HttpRequest * request_ = NULL; + HttpReply * reply_ = NULL; + if(!(request_ = dynamic_cast(adapted.header))) + { + request_ = (virgin.cause? virgin.cause: dynamic_cast(virgin.header)); + reply_ = dynamic_cast(adapted.header); + } + + Adaptation::Icap::History::Pointer h = request_->icapHistory(); + Must(h != NULL); // ICAPXaction::maybeLog calls only if there is a log + al.icp.opcode = ICP_INVALID; + al.url = h->log_uri.termedBuf(); + const Adaptation::Icap::ServiceRep &s = service(); + al.icap.reqMethod = s.cfg().method; + + al.cache.caddr = request_->client_addr; + + al.request = HTTPMSGLOCK(request_); + if(reply_) + al.reply = HTTPMSGLOCK(reply_); + else + al.reply = NULL; + + if (h->rfc931.size()) + al.cache.rfc931 = h->rfc931.termedBuf(); + +#if USE_SSL + if (h->ssluser.size()) + al.cache.ssluser = h->ssluser.termedBuf(); +#endif + al.cache.code = h->logType; + al.cache.requestSize = h->req_sz; + if (reply_) { + al.http.code = reply_->sline.status; + al.http.content_type = reply_->content_type.termedBuf(); + al.cache.replySize = replyBodySize + reply_->hdr_sz; + al.cache.highOffset = replyBodySize; + //don't set al.cache.objectSize because it hasn't exist yet + + Packer p; + MemBuf mb; + + mb.init(); + packerToMemInit(&p, &mb); + + reply_->header.packInto(&p); + al.headers.reply = xstrdup(mb.buf); + + packerClean(&p); + mb.clean(); + } + prepareLogWithRequestDetails(request_, &al); + Xaction::finalizeLogInfo(); +} + + void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf) { char ntoabuf[MAX_IPSTRLEN]; @@ -1048,6 +1169,21 @@ void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf) buf.Printf("Proxy-Authorization: " SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(vh)); } + const HttpRequest *request = &virginRequest(); + + // share the cross-transactional database records if needed + if (Adaptation::Config::masterx_shared_name) { + Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { + String name, value; + if (ah->getXxRecord(name, value)) { + buf.Printf(SQUIDSTRINGPH ": " SQUIDSTRINGPH "\r\n", + SQUIDSTRINGPRINT(name), SQUIDSTRINGPRINT(value)); + } + } + } + + buf.Printf("Encapsulated: "); MemBuf httpBuf; @@ -1057,10 +1193,6 @@ void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf) // build HTTP request header, if any ICAP::Method m = s.method; - const HttpRequest *request = virgin.cause ? - virgin.cause : - dynamic_cast(virgin.header); - // to simplify, we could assume that request is always available String urlPath; @@ -1116,6 +1248,9 @@ void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf) // start ICAP request body with encapsulated HTTP headers buf.append(httpBuf.content(), httpBuf.contentSize()); + // TODO: write IcapRequest class? + icapRequest->parseHeader(buf.content(),buf.contentSize()); + httpBuf.clean(); } @@ -1186,10 +1321,7 @@ void Adaptation::Icap::ModXact::decideOnPreview() return; } - const HttpRequest *request = virgin.cause ? - virgin.cause : - dynamic_cast(virgin.header); - const String urlPath = request ? request->urlpath : String(); + const String urlPath = virginRequest().urlpath; size_t wantedSize; if (!service().wantsPreview(urlPath, wantedSize)) { debugs(93, 5, HERE << "should not offer preview for " << urlPath); @@ -1531,6 +1663,7 @@ Adaptation::Icap::ModXactLauncher::ModXactLauncher(Adaptation::Initiator *anInit { virgin.setHeader(virginHeader); virgin.setCause(virginCause); + updateHistory(true); } Adaptation::Icap::Xaction *Adaptation::Icap::ModXactLauncher::createXaction() @@ -1540,3 +1673,25 @@ Adaptation::Icap::Xaction *Adaptation::Icap::ModXactLauncher::createXaction() Must(s != NULL); return new Adaptation::Icap::ModXact(this, virgin.header, virgin.cause, s); } + +void Adaptation::Icap::ModXactLauncher::swanSong() { + debugs(93, 5, HERE << "swan sings"); + updateHistory(false); + Adaptation::Icap::Launcher::swanSong(); +} + +void Adaptation::Icap::ModXactLauncher::updateHistory(bool start) { + HttpRequest *r = virgin.cause ? + virgin.cause : dynamic_cast(virgin.header); + + // r should never be NULL but we play safe; TODO: add Should() + if (r) { + Adaptation::Icap::History::Pointer h = r->icapHistory(); + if (h != NULL) { + if (start) + h->start("ICAPModXactLauncher"); + else + h->stop("ICAPModXactLauncher"); + } + } +} diff --git a/src/adaptation/icap/ModXact.h b/src/adaptation/icap/ModXact.h index f0e23cede3..9590ca7aa3 100644 --- a/src/adaptation/icap/ModXact.h +++ b/src/adaptation/icap/ModXact.h @@ -167,6 +167,9 @@ protected: private: virtual void start(); + /// locates the request, either as a cause or as a virgin message itself + const HttpRequest &virginRequest() const; // Must always be available + void estimateVirginBody(); void makeAdaptedBodyPipe(const char *what); @@ -250,8 +253,7 @@ private: bool gotEncapsulated(const char *section) const; void checkConsuming(); - - HttpReply *icapReply; + virtual void finalizeLogInfo(); SizedEstimate virginBody; VirginBodyAct virginBodyWriting; // virgin body writing state @@ -263,6 +265,10 @@ private: bool canStartBypass; // enables bypass of transaction failures + uint64_t replyBodySize; ///< dechunked ICAP reply body size + + int adaptHistoryId; ///< adaptation history slot reservation + class State { @@ -319,6 +325,11 @@ public: protected: virtual Xaction *createXaction(); + virtual void swanSong(); + + /// starts or stops transaction accounting in ICAP history + void updateHistory(bool start); + InOut virgin; private: diff --git a/src/adaptation/icap/OptXact.cc b/src/adaptation/icap/OptXact.cc index 4696945efd..5219cc9b0a 100644 --- a/src/adaptation/icap/OptXact.cc +++ b/src/adaptation/icap/OptXact.cc @@ -9,6 +9,8 @@ #include "adaptation/icap/OptXact.h" #include "adaptation/icap/Options.h" #include "TextException.h" +#include "SquidTime.h" +#include "HttpRequest.h" CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap, OptXact); CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap, OptXactLauncher); @@ -36,7 +38,7 @@ void Adaptation::Icap::OptXact::handleCommConnected() makeRequest(requestBuf); debugs(93, 9, HERE << "request " << status() << ":\n" << (requestBuf.terminate(), requestBuf.content())); - + icap_tio_start = current_time; scheduleWrite(requestBuf); } @@ -48,6 +50,10 @@ void Adaptation::Icap::OptXact::makeRequest(MemBuf &buf) const String host = s.cfg().host; buf.Printf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(host), s.cfg().port); buf.append(ICAP::crlf, 2); + + // XXX: HttpRequest cannot fully parse ICAP Request-Line + http_status status; + Must(icapRequest->parse(&buf, true, &status) > 0); } void Adaptation::Icap::OptXact::handleCommWrote(size_t size) @@ -60,7 +66,10 @@ void Adaptation::Icap::OptXact::handleCommWrote(size_t size) void Adaptation::Icap::OptXact::handleCommRead(size_t) { if (HttpMsg *r = parseResponse()) { + icap_tio_finish = current_time; + setOutcome(xoOpt); sendAnswer(r); + icapReply = HTTPMSGLOCK(dynamic_cast(r)); Must(done()); // there should be nothing else to do return; } @@ -74,11 +83,11 @@ HttpMsg *Adaptation::Icap::OptXact::parseResponse() status()); debugs(93, 5, HERE << "\n" << readBuf.content()); - HttpReply *r = new HttpReply; + HttpReply *r = HTTPMSGLOCK(new HttpReply); r->protoPrefix = "ICAP/"; // TODO: make an IcapReply class? if (!parseHttpMsg(r)) { // throws on errors - delete r; + HTTPMSGUNLOCK(r); return 0; } @@ -88,6 +97,18 @@ HttpMsg *Adaptation::Icap::OptXact::parseResponse() return r; } +void Adaptation::Icap::OptXact::swanSong() +{ + Adaptation::Icap::Xaction::swanSong(); +} + +void Adaptation::Icap::OptXact::finalizeLogInfo() +{ + // al.cache.caddr = 0; + al.icap.reqMethod = Adaptation::methodOptions; + Adaptation::Icap::Xaction::finalizeLogInfo(); +} + /* Adaptation::Icap::OptXactLauncher */ Adaptation::Icap::OptXactLauncher::OptXactLauncher(Adaptation::Initiator *anInitiator, Adaptation::ServicePointer aService): diff --git a/src/adaptation/icap/OptXact.h b/src/adaptation/icap/OptXact.h index d11e581860..b418f05edc 100644 --- a/src/adaptation/icap/OptXact.h +++ b/src/adaptation/icap/OptXact.h @@ -63,7 +63,10 @@ protected: void startReading(); + virtual void swanSong(); + private: + virtual void finalizeLogInfo(); CBDATA_CLASS2(OptXact); }; diff --git a/src/adaptation/icap/Xaction.cc b/src/adaptation/icap/Xaction.cc index bb64793295..c4bfbc3fc6 100644 --- a/src/adaptation/icap/Xaction.cc +++ b/src/adaptation/icap/Xaction.cc @@ -7,10 +7,16 @@ #include "CommCalls.h" #include "HttpMsg.h" #include "adaptation/icap/Xaction.h" +#include "adaptation/icap/Launcher.h" #include "adaptation/icap/Config.h" #include "TextException.h" #include "pconn.h" +#include "HttpRequest.h" +#include "HttpReply.h" +#include "acl/FilledChecklist.h" +#include "icap_log.h" #include "fde.h" +#include "SquidTime.h" static PconnPool *icapPconnPool = new PconnPool("ICAP Servers"); @@ -20,22 +26,30 @@ static PconnPool *icapPconnPool = new PconnPool("ICAP Servers"); Adaptation::Icap::Xaction::Xaction(const char *aTypeName, Adaptation::Initiator *anInitiator, Adaptation::Icap::ServiceRep::Pointer &aService): AsyncJob(aTypeName), Adaptation::Initiate(aTypeName, anInitiator, aService.getRaw()), + icapRequest(NULL), + icapReply(NULL), + attempts(0), connection(-1), commBuf(NULL), commBufSize(0), commEof(false), reuseConnection(true), isRetriable(true), + isRepeatable(true), ignoreLastWrite(false), connector(NULL), reader(NULL), writer(NULL), closer(NULL) { debugs(93,3, typeName << " constructed, this=" << this << " [icapx" << id << ']'); // we should not call virtual status() here + icapRequest = HTTPMSGLOCK(new HttpRequest); + icap_tr_start = current_time; } Adaptation::Icap::Xaction::~Xaction() { debugs(93,3, typeName << " destructed, this=" << this << " [icapx" << id << ']'); // we should not call virtual status() here + HTTPMSGUNLOCK(icapRequest); + HTTPMSGUNLOCK(icapReply); } Adaptation::Icap::ServiceRep & @@ -48,11 +62,18 @@ Adaptation::Icap::Xaction::service() void Adaptation::Icap::Xaction::disableRetries() { - debugs(93,5, typeName << (isRetriable ? " becomes" : " remains") << - " final" << status()); + debugs(93,5, typeName << (isRetriable ? " from now on" : " still") << + " cannot be retried " << status()); isRetriable = false; } +void Adaptation::Icap::Xaction::disableRepeats(const char *reason) +{ + debugs(93,5, typeName << (isRepeatable ? " from now on" : " still") << + " cannot be repeated because " << reason << status()); + isRepeatable = false; +} + void Adaptation::Icap::Xaction::start() { Adaptation::Initiate::start(); @@ -220,6 +241,7 @@ void Adaptation::Icap::Xaction::noteCommWrote(const CommIoCbParams &io) debugs(93, 7, HERE << "ignoring last write; status: " << io.flag); } else { Must(io.flag == COMM_OK); + al.icap.bytesSent += io.size; updateTimeout(); handleCommWrote(io.size); } @@ -256,6 +278,13 @@ void Adaptation::Icap::Xaction::handleCommClosed() mustStop("ICAP service connection externally closed"); } +void Adaptation::Icap::Xaction::callException(const std::exception &e) +{ + setOutcome(xoError); + Adaptation::Initiate::callException(e); +} + + void Adaptation::Icap::Xaction::callEnd() { if (doneWithIo()) { @@ -317,6 +346,8 @@ void Adaptation::Icap::Xaction::noteCommRead(const CommIoCbParams &io) Must(io.flag == COMM_OK); Must(io.size >= 0); + al.icap.bytesRead+=io.size; + updateTimeout(); debugs(93, 3, HERE << "read " << io.size << " bytes"); @@ -397,6 +428,17 @@ void Adaptation::Icap::Xaction::noteInitiatorAborted() } +void Adaptation::Icap::Xaction::setOutcome(const Adaptation::Icap::XactOutcome &xo) +{ + if (al.icap.outcome != xoUnknown) { + debugs(93, 3, HERE << "Warning: reseting outcome: from " << + al.icap.outcome << " to " << xo); + } else { + debugs(93, 4, HERE << xo); + } + al.icap.outcome = xo; +} + // This 'last chance' method is called before a 'done' transaction is deleted. // It is wrong to call virtual methods from a destructor. Besides, this call // indicates that the transaction will terminate as planned. @@ -413,11 +455,56 @@ void Adaptation::Icap::Xaction::swanSong() memFreeBuf(commBufSize, commBuf); if (theInitiator) - tellQueryAborted(!isRetriable); + tellQueryAborted(); + + maybeLog(); Adaptation::Initiate::swanSong(); } +void Adaptation::Icap::Xaction::tellQueryAborted() { + Adaptation::Icap::Launcher *l = dynamic_cast(theInitiator.ptr()); + Adaptation::Icap::XactAbortInfo abortInfo(icapRequest, icapReply, retriable(), repeatable()); + CallJob(91, 5, __FILE__, __LINE__, + "Adaptation::Icap::Launcher::noteXactAbort", + XactAbortCall(l, &Adaptation::Icap::Launcher::noteXactAbort, abortInfo) ); + clearInitiator(); +} + + +void Adaptation::Icap::Xaction::maybeLog() { + if(IcapLogfileStatus == LOG_ENABLE) + { + ACLChecklist *checklist = new ACLFilledChecklist(::Config.accessList.icap, al.request, dash_str); + if (!::Config.accessList.icap || checklist->fastCheck()) { + finalizeLogInfo(); + icapLogLog(&al, checklist); + } + accessLogFreeMemory(&al); + delete checklist; + } +} + +void Adaptation::Icap::Xaction::finalizeLogInfo() +{ + //prepare log data + al.icp.opcode = ICP_INVALID; + + const Adaptation::Icap::ServiceRep &s = service(); + al.icap.hostAddr = s.cfg().host.termedBuf(); + al.icap.serviceName = s.cfg().key; + al.icap.reqUri = s.cfg().uri; + + al.icap.ioTime = tvSubMsec(icap_tio_start, icap_tio_finish); + al.icap.trTime = tvSubMsec(icap_tr_start, current_time); + + al.icap.request = HTTPMSGLOCK(icapRequest); + if (icapReply) { + al.icap.reply = HTTPMSGLOCK(icapReply); + al.icap.resStatus = icapReply->sline.status; + } +} + // returns a temporary string depicting transaction status, for debugging const char *Adaptation::Icap::Xaction::status() const { diff --git a/src/adaptation/icap/Xaction.h b/src/adaptation/icap/Xaction.h index 3ce2002c64..6fc7931688 100644 --- a/src/adaptation/icap/Xaction.h +++ b/src/adaptation/icap/Xaction.h @@ -39,6 +39,7 @@ #include "MemBuf.h" #include "adaptation/icap/ServiceRep.h" #include "adaptation/Initiate.h" +#include "AccessLogEntry.h" class HttpMsg; class CommConnectCbParams; @@ -65,6 +66,9 @@ public: virtual ~Xaction(); void disableRetries(); + void disableRepeats(const char *reason); + bool retriable() const { return isRetriable; } + bool repeatable() const { return isRepeatable; } // comm handler wrappers, treat as private void noteCommConnected(const CommConnectCbParams &io); @@ -73,6 +77,13 @@ public: void noteCommTimedout(const CommTimeoutCbParams &io); void noteCommClosed(const CommCloseCbParams &io); + // TODO: create these only when actually sending/receiving + HttpRequest *icapRequest; ///< sent (or at least created) ICAP request + HttpReply *icapReply; ///< received ICAP reply, if any + + /// the number of times we tried to get to the service, including this time + int attempts; + protected: virtual void start(); virtual void noteInitiatorAborted(); // TODO: move to Adaptation::Initiate @@ -113,11 +124,20 @@ protected: // useful for debugging virtual bool fillVirginHttpHeader(MemBuf&) const; - // custom end-of-call checks + // custom exception handling and end-of-call checks + virtual void callException(const std::exception &e); virtual void callEnd(); + // logging + void setOutcome(const XactOutcome &xo); + virtual void finalizeLogInfo(); + ServiceRep &service(); +private: + void tellQueryAborted(); + void maybeLog(); + protected: int connection; // FD of the ICAP server connection @@ -137,7 +157,8 @@ protected: size_t commBufSize; bool commEof; bool reuseConnection; - bool isRetriable; + bool isRetriable; ///< can retry on persistent connection failures + bool isRepeatable; ///< can repeat if no or unsatisfactory response bool ignoreLastWrite; const char *stopReason; @@ -148,6 +169,12 @@ protected: AsyncCall::Pointer writer; AsyncCall::Pointer closer; + AccessLogEntry al; + + timeval icap_tr_start; /*time when the ICAP transaction was created */ + timeval icap_tio_start; /*time when the first ICAP request byte was scheduled for sending*/ + timeval icap_tio_finish; /*time when the last byte of the ICAP responsewas received*/ + private: //CBDATA_CLASS2(Xaction); }; diff --git a/src/adaptation/icap/icap_log.cc b/src/adaptation/icap/icap_log.cc new file mode 100644 index 0000000000..e1aa754791 --- /dev/null +++ b/src/adaptation/icap/icap_log.cc @@ -0,0 +1,52 @@ +#include "squid.h" +#include "icap_log.h" +#include "AccessLogEntry.h" + +int IcapLogfileStatus = LOG_DISABLE; + +void +icapLogOpen() +{ + customlog *log; + + for (log = Config.Log.icaplogs; log; log = log->next) { + if (log->type == CLF_NONE) + continue; + + if (log->type == CLF_AUTO) + log->type = CLF_ICAP_SQUID; + + log->logfile = logfileOpen(log->filename, MAX_URL << 1, 1); + + IcapLogfileStatus = LOG_ENABLE; + } +} + +void +icapLogClose() +{ + customlog *log; + + for (log = Config.Log.icaplogs; log; log = log->next) { + if (log->logfile) { + logfileClose(log->logfile); + log->logfile = NULL; + } + } +} + +void +icapLogRotate() +{ + for (customlog* log = Config.Log.icaplogs; log; log = log->next) { + if (log->logfile) { + logfileRotate(log->logfile); + } + } +} + +void icapLogLog(AccessLogEntry *al, ACLChecklist * checklist) +{ + if (IcapLogfileStatus == LOG_ENABLE) + accessLogLogTo(Config.Log.icaplogs, al, checklist); +} diff --git a/src/adaptation/icap/icap_log.h b/src/adaptation/icap/icap_log.h new file mode 100644 index 0000000000..bf577ab399 --- /dev/null +++ b/src/adaptation/icap/icap_log.h @@ -0,0 +1,14 @@ +#ifndef ICAP_LOG_H_ +#define ICAP_LOG_H_ + +class AccessLogEntry; +class ACLChecklist; + +void icapLogClose(); +void icapLogOpen(); +void icapLogRotate(); +void icapLogLog(AccessLogEntry *al, ACLChecklist * checklist); + +extern int IcapLogfileStatus; + +#endif /*ICAP_LOG_H_*/ diff --git a/src/cache_cf.cc b/src/cache_cf.cc index 536234ad91..7400f84375 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -3432,6 +3432,10 @@ parse_access_log(customlog ** logs) cl->type = CLF_SQUID; } else if (strcmp(logdef_name, "common") == 0) { cl->type = CLF_COMMON; +#if ICAP_CLIENT + } else if (strcmp(logdef_name, "icap_squid") == 0) { + cl->type = CLF_ICAP_SQUID; +#endif } else { debugs(3, 0, "Log format '" << logdef_name << "' is not defined"); self_destruct(); @@ -3484,7 +3488,11 @@ dump_access_log(StoreEntry * entry, const char *name, customlog * logs) case CLF_COMMON: storeAppendPrintf(entry, "%s squid", log->filename); break; - +#if ICAP_CLIENT + case CLF_ICAP_SQUID: + storeAppendPrintf(entry, "%s icap_squid", log->filename); + break; +#endif case CLF_AUTO: if (log->aclList) diff --git a/src/cf.data.pre b/src/cf.data.pre index c918e394e8..f1d81f1c89 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -2312,6 +2312,7 @@ DOC_START Format codes: + % a literal % character >a Client source IP address >A Client FQDN >p Client source port @@ -2321,42 +2322,96 @@ DOC_START ts Seconds since epoch tu subsecond time (milliseconds) tl Local time. Optional strftime format argument - default %d/%b/%Y:%H:%M:%S %z + default %d/%b/%Y:%H:%M:%S %z tg GMT time. Optional strftime format argument - default %d/%b/%Y:%H:%M:%S %z + default %d/%b/%Y:%H:%M:%S %z tr Response time (milliseconds) - >h Request header. Optional header name argument - on the format header[:[separator]element] - h - un User name - ul User name from authentication - ui User name from ident - us User name from SSL - ue User name from external acl helper - Hs HTTP status code - Ss Squid request status (TCP_MISS etc) - Sh Squid hierarchy status (DEFAULT_PARENT etc) - mt MIME content type - rm Request method (GET/POST etc) - ru Request URL - rp Request URL-Path excluding hostname - rv Request protocol version - et Tag returned by external acl - ea Log string returned by external acl - st Request size including HTTP headers - st Request+Reply size including HTTP headers - h Request header. Optional header name argument + on the format header[:[separator]element] + [http::]h + [http::]un User name + [http::]ul User name from authentication + [http::]ui User name from ident + [http::]us User name from SSL + [http::]ue User name from external acl helper + [http::]>Hs HTTP status code sent to the client + [http::]st Received request size including HTTP headers. In the + case of chunked requests the chunked encoding metadata + are not included + [http::]>sh Received HTTP request headers size + [http::]a %Ss/%03Hs %a %Ss/%03Hs %h] [%a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %h" "%{User-Agent}>h" %Ss:%Sh +logformat squid %ts.%03tu %6tr %>a %Ss/%03>Hs %a %Ss/%03>Hs %h] [%a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh DOC_END NAME: access_log cache_access_log @@ -2391,6 +2446,88 @@ DOC_START access_log @DEFAULT_ACCESS_LOG@ squid DOC_END +NAME: icap_log +TYPE: access_log +IFDEF: ICAP_CLIENT +LOC: Config.Log.icaplogs +DEFAULT: none +DOC_START + ICAP log files record ICAP transaction summaries, one line per + transaction. + + The icap_log option format is: + icap_log [ [acl acl ...]] + icap_log none [acl acl ...]] + + Please see access_log option documentation for details. The two + kinds of logs share the overall configuration approach and many + features. + + ICAP processing of a single HTTP message or transaction may + require multiple ICAP transactions. In such cases, multiple + ICAP transaction log lines will correspond to a single access + log line. + + ICAP log uses logformat codes that make sense for an ICAP + transaction. Header-related codes are applied to the HTTP header + embedded in an ICAP server response, with the following caveats: + For REQMOD, there is no HTTP response header unless the ICAP + server performed request satisfaction. For RESPMOD, the HTTP + request header is the header sent to the ICAP server. For + OPTIONS, there are no HTTP headers. + + The following format codes are also available for ICAP logs: + + icap::st Bytes sent to the ICAP server (TCP payload + only; i.e., what Squid writes to the socket). + + icap::h ICAP request header(s). Similar to >h. + + icap::a %icap::to/%03icap::Hs %icap::icapHistory(); +#endif if (Config.onoff.log_mime_hdrs) { Packer p; MemBuf mb; @@ -456,14 +460,31 @@ clientPrepareLogWithRequestDetails(HttpRequest * request, AccessLogEntry * aLogE packerToMemInit(&p, &mb); request->header.packInto(&p); aLogEntry->headers.request = xstrdup(mb.buf); + +#if ICAP_CLIENT + packerClean(&p); + mb.reset(); + packerToMemInit(&p, &mb); + + if (ih != NULL) + ih->lastIcapHeader.packInto(&p); + aLogEntry->headers.icap = xstrdup(mb.buf); +#endif + packerClean(&p); mb.clean(); } +#if ICAP_CLIENT + if (ih != NULL) + aLogEntry->icap.processingTime = ih->processingTime(); +#endif + aLogEntry->http.method = request->method; aLogEntry->http.version = request->http_ver; aLogEntry->hier = request->hier; - aLogEntry->cache.requestSize += request->content_length; + if (request->content_length > 0) // negative when no body or unknown length + aLogEntry->cache.requestSize += request->content_length; aLogEntry->cache.extuser = request->extacl_user.termedBuf(); if (request->auth_user_request) { @@ -502,8 +523,10 @@ ClientHttpRequest::logRequest() if (getConn() != NULL) al.cache.caddr = getConn()->log_addr; al.cache.requestSize = req_sz; + al.cache.requestHeadersSize = req_sz; al.cache.replySize = out.size; + al.cache.replyHeadersSize = out.headers_sz; al.cache.highOffset = out.offset; @@ -512,7 +535,7 @@ ClientHttpRequest::logRequest() al.cache.msec = tvSubMsec(start_time, current_time); if (request) - clientPrepareLogWithRequestDetails(request, &al); + prepareLogWithRequestDetails(request, &al); if (getConn() != NULL && getConn()->rfc931[0]) al.cache.rfc931 = getConn()->rfc931; @@ -1873,6 +1896,18 @@ prepareTransparentURL(ConnStateData * conn, ClientHttpRequest *http, char *url, } } +// Temporary hack helper: determine whether the request is chunked, expensive +static bool +isChunkedRequest(const HttpParser *hp) { + HttpRequest request; + if (!request.parseHeader(HttpParserHdrBuf(hp), HttpParserHdrSz(hp))) + return false; + + return request.header.has(HDR_TRANSFER_ENCODING) && + request.header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ','); +} + + /** * parseHttpRequest() * @@ -1885,7 +1920,6 @@ prepareTransparentURL(ConnStateData * conn, ClientHttpRequest *http, char *url, static ClientSocketContext * parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method_p, HttpVersion *http_ver) { - char *url = NULL; char *req_hdr = NULL; char *end; size_t req_sz; @@ -1952,17 +1986,6 @@ parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method return parseHttpRequestAbort(conn, "error:unsupported-request-method"); } - /* set url */ - /* - * XXX this should eventually not use a malloc'ed buffer; the transformation code - * below needs to be modified to not expect a mutable nul-terminated string. - */ - url = (char *)xmalloc(hp->u_end - hp->u_start + 16); - - memcpy(url, hp->buf + hp->u_start, hp->u_end - hp->u_start + 1); - - url[hp->u_end - hp->u_start + 1] = '\0'; - /* * Process headers after request line * TODO: Use httpRequestParse here. @@ -1982,7 +2005,6 @@ parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method */ if ( squid_strnstr(req_hdr, "\r\r\n", req_sz) ) { debugs(33, 1, "WARNING: suspicious HTTP request contains double CR"); - xfree(url); return parseHttpRequestAbort(conn, "error:double-CR"); } @@ -1990,6 +2012,35 @@ parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method (int) HttpParserRequestLen(hp) << ", req_line_sz = " << HttpParserReqSz(hp)); + // Temporary hack: We might receive a chunked body from a broken HTTP/1.1 + // client that sends chunked requests to HTTP/1.0 Squid. If the request + // might have a chunked body, parse the headers early to look for the + // "Transfer-Encoding: chunked" header. If we find it, wait until the + // entire body is available so that we can set the content length and + // forward the request without chunks. The primary reason for this is + // to avoid forwarding a chunked request because the server side lacks + // logic to determine when it is valid to do so. + // FUTURE_CODE_TO_SUPPORT_CHUNKED_REQUESTS below will replace this hack. + if (hp->v_min == 1 && hp->v_maj == 1 && // broken client, may send chunks + Config.maxChunkedRequestBodySize > 0 && // configured to dechunk + (*method_p == METHOD_PUT || *method_p == METHOD_POST)) { + + // check only once per request because isChunkedRequest is expensive + if (conn->in.dechunkingState == ConnStateData::chunkUnknown) { + if (isChunkedRequest(hp)) + conn->startDechunkingRequest(hp); + else + conn->in.dechunkingState = ConnStateData::chunkNone; + } + + if (conn->in.dechunkingState == ConnStateData::chunkParsing) { + if (conn->parseRequestChunks(hp)) // parses newly read chunks + return NULL; // wait for more data + debugs(33, 5, HERE << "Got complete chunked request or err."); + assert(conn->in.dechunkingState != ConnStateData::chunkParsing); + } + } + /* Ok, all headers are received */ http = new ClientHttpRequest(conn); @@ -2006,6 +2057,17 @@ parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method debugs(33, 5, "parseHttpRequest: Request Header is\n" <<(hp->buf) + hp->hdr_start); + /* set url */ + /* + * XXX this should eventually not use a malloc'ed buffer; the transformation code + * below needs to be modified to not expect a mutable nul-terminated string. + */ + char *url = (char *)xmalloc(hp->u_end - hp->u_start + 16); + + memcpy(url, hp->buf + hp->u_start, hp->u_end - hp->u_start + 1); + + url[hp->u_end - hp->u_start + 1] = '\0'; + #if THIS_VIOLATES_HTTP_SPECS_ON_URL_TRANSFORMATION if ((t = strchr(url, '#'))) /* remove HTML anchors */ @@ -2157,6 +2219,11 @@ connNoteUseOfBuffer(ConnStateData* conn, size_t byteCount) int connKeepReadingIncompleteRequest(ConnStateData * conn) { + // when we read chunked requests, the entire body is buffered + // XXX: this check ignores header size and its limits. + if (conn->in.dechunkingState == ConnStateData::chunkParsing) + return conn->in.notYetUsed < Config.maxChunkedRequestBodySize; + return conn->in.notYetUsed >= Config.maxRequestHeaderSize ? 0 : 1; } @@ -2166,8 +2233,13 @@ connCancelIncompleteRequests(ConnStateData * conn) ClientSocketContext *context = parseHttpRequestAbort(conn, "error:request-too-large"); clientStreamNode *node = context->getClientReplyContext(); assert(!connKeepReadingIncompleteRequest(conn)); - debugs(33, 1, "Request header is too large (" << conn->in.notYetUsed << " bytes)"); - debugs(33, 1, "Config 'request_header_max_size'= " << Config.maxRequestHeaderSize << " bytes."); + if (conn->in.dechunkingState == ConnStateData::chunkParsing) { + debugs(33, 1, "Chunked request is too large (" << conn->in.notYetUsed << " bytes)"); + debugs(33, 1, "Config 'chunked_request_body_max_size'= " << Config.maxChunkedRequestBodySize << " bytes."); + } else { + debugs(33, 1, "Request header is too large (" << conn->in.notYetUsed << " bytes)"); + debugs(33, 1, "Config 'request_header_max_size'= " << Config.maxRequestHeaderSize << " bytes."); + } clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); assert (repContext); repContext->setReplyToError(ERR_TOO_BIG, @@ -2213,6 +2285,9 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c ClientHttpRequest *http = context->http; HttpRequest *request = NULL; bool notedUseOfBuffer = false; + bool tePresent = false; + bool deChunked = false; + bool unsupportedTe = false; /* We have an initial client stream in place should it be needed */ /* setup our private context */ @@ -2313,8 +2388,17 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c request->my_addr = conn->me; request->http_ver = http_ver; - if (!urlCheckRequest(request) || - request->header.has(HDR_TRANSFER_ENCODING)) { + tePresent = request->header.has(HDR_TRANSFER_ENCODING); + deChunked = conn->in.dechunkingState == ConnStateData::chunkReady; + if (deChunked) { + assert(tePresent); + request->setContentLength(conn->in.dechunked.contentSize()); + request->header.delById(HDR_TRANSFER_ENCODING); + conn->finishDechunkingRequest(hp); + } + + unsupportedTe = tePresent && !deChunked; + if (!urlCheckRequest(request) || unsupportedTe) { clientStreamNode *node = context->getClientReplyContext(); clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); assert (repContext); @@ -2648,13 +2732,74 @@ ConnStateData::handleRequestBodyData() { assert(bodyPipe != NULL); - if (const size_t putSize = bodyPipe->putMoreData(in.buf, in.notYetUsed)) - connNoteUseOfBuffer(this, putSize); + size_t putSize = 0; + +#if FUTURE_CODE_TO_SUPPORT_CHUNKED_REQUESTS + // The code below works, in principle, but we cannot do dechunking + // on-the-fly because that would mean sending chunked requests to + // the next hop. Squid lacks logic to determine which servers can + // receive chunk requests. Squid v3.0 code cannot even handle chunked + // responses which we may encourage by sending chunked requests. + // The error generation code probably needs more work. + if (in.bodyParser) { // chunked body + debugs(33,5, HERE << "handling chunked request body for FD " << fd); + bool malformedChunks = false; + + MemBuf raw; // ChunkedCodingParser only works with MemBufs + raw.init(in.notYetUsed, in.notYetUsed); + raw.append(in.buf, in.notYetUsed); + try { // the parser will throw on errors + const mb_size_t wasContentSize = raw.contentSize(); + BodyPipeCheckout bpc(*bodyPipe); + const bool parsed = in.bodyParser->parse(&raw, &bpc.buf); + bpc.checkIn(); + putSize = wasContentSize - raw.contentSize(); + + if (parsed) { + stopProducingFor(bodyPipe, true); // this makes bodySize known + } else { + // parser needy state must imply body pipe needy state + if (in.bodyParser->needsMoreData() && + !bodyPipe->mayNeedMoreData()) + malformedChunks = true; + // XXX: if bodyParser->needsMoreSpace, how can we guarantee it? + } + } catch (...) { // XXX: be more specific + malformedChunks = true; + } - if (!bodyPipe->mayNeedMoreData()) { - // BodyPipe will clear us automagically when we produced everything - bodyPipe = NULL; + if (malformedChunks) { + if (bodyPipe != NULL) + stopProducingFor(bodyPipe, false); + + ClientSocketContext::Pointer context = getCurrentContext(); + if (!context->http->out.offset) { + clientStreamNode *node = context->getClientReplyContext(); + clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); + assert (repContext); + repContext->setReplyToError(ERR_INVALID_REQ, HTTP_BAD_REQUEST, + METHOD_NONE, NULL, &peer.sin_addr, + NULL, NULL, NULL); + context->pullData(); + } + flags.readMoreRequests = false; + return; // XXX: is that sufficient to generate an error? + } + } else // identity encoding +#endif + { + debugs(33,5, HERE << "handling plain request body for FD " << fd); + putSize = bodyPipe->putMoreData(in.buf, in.notYetUsed); + if (!bodyPipe->mayNeedMoreData()) { + // BodyPipe will clear us automagically when we produced everything + bodyPipe = NULL; + } + } + + if (putSize > 0) + connNoteUseOfBuffer(this, putSize); + if (!bodyPipe) { debugs(33,5, HERE << "produced entire request body for FD " << fd); if (closing()) { @@ -3435,19 +3580,121 @@ ConnStateData::startClosing(const char *reason) bodyPipe->enableAutoConsumption(); } +// initialize dechunking state +void +ConnStateData::startDechunkingRequest(HttpParser *hp) +{ + debugs(33, 5, HERE << "start dechunking at " << HttpParserRequestLen(hp)); + assert(in.dechunkingState == chunkUnknown); + assert(!in.bodyParser); + in.bodyParser = new ChunkedCodingParser; + in.chunkedSeen = HttpParserRequestLen(hp); // skip headers when dechunking + in.chunked.init(); // TODO: should we have a smaller-than-default limit? + in.dechunked.init(); + in.dechunkingState = chunkParsing; +} + +// put parsed content into input buffer and clean up +void +ConnStateData::finishDechunkingRequest(HttpParser *hp) +{ + debugs(33, 5, HERE << "finish dechunking; content: " << in.dechunked.contentSize()); + + assert(in.dechunkingState == chunkReady); + assert(in.bodyParser); + delete in.bodyParser; + in.bodyParser = NULL; + + const mb_size_t headerSize = HttpParserRequestLen(hp); + + // dechunking cannot make data bigger + assert(headerSize + in.dechunked.contentSize() + in.chunked.contentSize() + <= static_cast(in.notYetUsed)); + assert(in.notYetUsed <= in.allocatedSize); + + // copy dechunked content + char *end = in.buf + headerSize; + xmemmove(end, in.dechunked.content(), in.dechunked.contentSize()); + end += in.dechunked.contentSize(); + + // copy post-chunks leftovers, if any, caused by request pipelining? + if (in.chunked.contentSize()) { + xmemmove(end, in.chunked.content(), in.chunked.contentSize()); + end += in.chunked.contentSize(); + } + + in.notYetUsed = end - in.buf; + + in.chunked.clean(); + in.dechunked.clean(); + in.dechunkingState = chunkUnknown; +} + +// parse newly read request chunks and buffer them for finishDechunkingRequest +// returns true iff needs more data +bool +ConnStateData::parseRequestChunks(HttpParser *) +{ + debugs(33,5, HERE << "parsing chunked request body at " << + in.chunkedSeen << " < " << in.notYetUsed); + assert(in.bodyParser); + assert(in.dechunkingState == chunkParsing); + + assert(in.chunkedSeen <= in.notYetUsed); + const mb_size_t fresh = in.notYetUsed - in.chunkedSeen; + + // be safe: count some chunked coding metadata towards the total body size + if (fresh + in.dechunked.contentSize() > Config.maxChunkedRequestBodySize) { + debugs(33,3, HERE << "chunked body (" << fresh << " + " << + in.dechunked.contentSize() << " may exceed " << + "chunked_request_body_max_size=" << + Config.maxChunkedRequestBodySize); + in.dechunkingState = chunkError; + return false; + } + + if (fresh > in.chunked.potentialSpaceSize()) { + // should not happen if Config.maxChunkedRequestBodySize is reasonable + debugs(33,1, HERE << "request_body_max_size exceeds chunked buffer " << + "size: " << fresh << " + " << in.chunked.contentSize() << " > " << + in.chunked.potentialSpaceSize() << " with " << + "chunked_request_body_max_size=" << + Config.maxChunkedRequestBodySize); + in.dechunkingState = chunkError; + return false; + } + in.chunked.append(in.buf + in.chunkedSeen, fresh); + in.chunkedSeen += fresh; + + try { // the parser will throw on errors + if (in.bodyParser->parse(&in.chunked, &in.dechunked)) + in.dechunkingState = chunkReady; // successfully parsed all chunks + else + return true; // need more, keep the same state + } catch (...) { + debugs(33,3, HERE << "chunk parsing error"); + in.dechunkingState = chunkError; + } + return false; // error, unsupported, or done +} + char * ConnStateData::In::addressToReadInto() const { return buf + notYetUsed; } -ConnStateData::In::In() : buf (NULL), notYetUsed (0), allocatedSize (0) +ConnStateData::In::In() : bodyParser(NULL), + buf (NULL), notYetUsed (0), allocatedSize (0), + dechunkingState(ConnStateData::chunkUnknown) {} ConnStateData::In::~In() { if (allocatedSize) memFreeBuf(allocatedSize, buf); + if (bodyParser) + delete bodyParser; // TODO: pool } /* This is a comm call normally scheduled by comm_close() */ diff --git a/src/client_side.h b/src/client_side.h index 37ad29978e..a40f7f528d 100644 --- a/src/client_side.h +++ b/src/client_side.h @@ -48,6 +48,9 @@ class clientStreamNode; class AuthUserRequest; +class ChunkedCodingParser; +class HttpParser; + template class Range; @@ -144,13 +147,22 @@ public: int fd; + /// chunk buffering and parsing algorithm state + typedef enum { chunkUnknown, chunkNone, chunkParsing, chunkReady, chunkError } DechunkingState; + struct In { In(); ~In(); char *addressToReadInto() const; + + ChunkedCodingParser *bodyParser; ///< parses chunked request body + MemBuf chunked; ///< contains unparsed raw (chunked) body data + MemBuf dechunked; ///< accumulates parsed (dechunked) content char *buf; size_t notYetUsed; size_t allocatedSize; + size_t chunkedSeen; ///< size of processed or ignored raw read data + DechunkingState dechunkingState; ///< request dechunking state } in; int64_t bodySizeLeft(); @@ -253,6 +265,10 @@ public: bool switchedToHttps() const { return false; } #endif + void startDechunkingRequest(HttpParser *hp); + bool parseRequestChunks(HttpParser *hp); + void finishDechunkingRequest(HttpParser *hp); + private: int connReadWasError(comm_err_t flag, int size, int xerrno); int connFinishedWithConn(int size); diff --git a/src/client_side_request.cc b/src/client_side_request.cc index cd51b70323..3b1430c3d9 100644 --- a/src/client_side_request.cc +++ b/src/client_side_request.cc @@ -59,13 +59,19 @@ #include "SquidTime.h" #include "wordlist.h" #include "inet_pton.h" +#include "fde.h" #if USE_ADAPTATION #include "adaptation/AccessCheck.h" #include "adaptation/Service.h" +#if ICAP_CLIENT +#include "adaptation/icap/History.h" +#endif static void adaptationAclCheckDoneWrapper(Adaptation::ServicePointer service, void *data); #endif + + #if LINGERING_CLOSE #define comm_close comm_lingering_close #endif @@ -629,6 +635,22 @@ ClientRequestContext::adaptationAclCheckDone(Adaptation::ServicePointer service) debugs(93,3,HERE << this << " adaptationAclCheckDone called"); assert(http); +#if ICAP_CLIENT + Adaptation::Icap::History::Pointer ih = http->request->icapHistory(); + if(ih != NULL) + { + if (http->getConn() != NULL) + { + ih->rfc931 = http->getConn()->rfc931; +#if USE_SSL + ih->ssluser = sslGetUserEmail(fd_table[http->getConn()->fd].ssl); +#endif + } + ih->log_uri = http->log_uri; + ih->req_sz = http->req_sz; + } +#endif + if (http->startAdaptation(service)) return; @@ -1301,6 +1323,12 @@ ClientHttpRequest::doCallouts() debugs(83, 3, HERE << "calling processRequest()"); processRequest(); + +#if ICAP_CLIENT + Adaptation::Icap::History::Pointer ih = request->icapHistory(); + if (ih != NULL) + ih->logType = logType; +#endif } #ifndef _USE_INLINE_ diff --git a/src/comm.cc b/src/comm.cc index d3a4294850..0f54519e69 100644 --- a/src/comm.cc +++ b/src/comm.cc @@ -209,6 +209,7 @@ public: IpAddress default_addr; // NP: CANNOT store the default addr:port together as it gets set/reset differently. + DnsLookupDetails dns; ///< host lookup details IpAddress S; AsyncCall::Pointer callback; @@ -876,19 +877,13 @@ commConnectStart(int fd, const char *host, u_short port, CNCB * callback, void * } static void -commConnectDnsHandle(const ipcache_addrs * ia, void *data) +commConnectDnsHandle(const ipcache_addrs *ia, const DnsLookupDetails &details, void *data) { ConnectStateData *cs = (ConnectStateData *)data; + cs->dns = details; if (ia == NULL) { debugs(5, 3, "commConnectDnsHandle: Unknown host: " << cs->host); - - if (!dns_error_message) { - dns_error_message = "Unknown DNS error"; - debugs(5, 1, "commConnectDnsHandle: Bad dns_error_message"); - } - - assert(dns_error_message != NULL); cs->callCallback(COMM_ERR_DNS, 0); return; } @@ -918,6 +913,7 @@ ConnectStateData::callCallback(comm_err_t status, int xerrno) typedef CommConnectCbParams Params; Params ¶ms = GetCommParams(callback); params.fd = fd; + params.dns = dns; params.flag = status; params.xerrno = xerrno; ScheduleCallHere(callback); diff --git a/src/comm.h b/src/comm.h index 730fba81e4..b15f8dfcd8 100644 --- a/src/comm.h +++ b/src/comm.h @@ -27,7 +27,8 @@ typedef enum { #endif } comm_err_t; -typedef void CNCB(int fd, comm_err_t status, int xerrno, void *data); +class DnsLookupDetails; +typedef void CNCB(int fd, const DnsLookupDetails &dns, comm_err_t status, int xerrno, void *data); typedef void IOCB(int fd, char *, size_t size, comm_err_t flag, int xerrno, void *data); diff --git a/src/dns_internal.cc b/src/dns_internal.cc index 72dada31a7..3d37e651ce 100644 --- a/src/dns_internal.cc +++ b/src/dns_internal.cc @@ -699,7 +699,7 @@ idnsDoSendQueryVC(nsvc *vc) } static void -idnsInitVCConnected(int fd, comm_err_t status, int xerrno, void *data) +idnsInitVCConnected(int fd, const DnsLookupDetails &, comm_err_t status, int xerrno, void *data) { nsvc * vc = (nsvc *)data; diff --git a/src/enums.h b/src/enums.h index e6649288c7..1de8ef8d93 100644 --- a/src/enums.h +++ b/src/enums.h @@ -533,6 +533,9 @@ typedef enum { CLF_CUSTOM, CLF_SQUID, CLF_COMMON, +#if ICAP_CLIENT + CLF_ICAP_SQUID, +#endif CLF_NONE } customlog_type; diff --git a/src/errorpage.cc b/src/errorpage.cc index d1e5a4aebc..a0b0891a69 100644 --- a/src/errorpage.cc +++ b/src/errorpage.cc @@ -496,7 +496,6 @@ errorStateFree(ErrorState * err) HTTPMSGUNLOCK(err->request); safe_free(err->redirect_url); safe_free(err->url); - safe_free(err->dnsserver_msg); safe_free(err->request_hdrs); wordlistDestroy(&err->ftp.server_msg); safe_free(err->ftp.request); @@ -536,9 +535,8 @@ ErrorState::Dump(MemBuf * mb) if (auth_user_request->denyMessage()) str.Printf("Auth ErrMsg: %s\r\n", auth_user_request->denyMessage()); - if (dnsserver_msg) { - str.Printf("DNS Server ErrMsg: %s\r\n", dnsserver_msg); - } + if (dnsError.size() > 0) + str.Printf("DNS ErrMsg: %s\r\n", dnsError.termedBuf()); /* - TimeStamp */ str.Printf("TimeStamp: %s\r\n\r\n", mkrfc1123(squid_curtime)); @@ -819,8 +817,8 @@ ErrorState::Convert(char token) break; case 'z': - if (dnsserver_msg) - p = dnsserver_msg; + if (dnsError.size() > 0) + p = dnsError.termedBuf(); else p = "[unknown]"; diff --git a/src/errorpage.h b/src/errorpage.h index b73564124f..d803780975 100644 --- a/src/errorpage.h +++ b/src/errorpage.h @@ -120,7 +120,7 @@ public: char *url; int xerrno; u_short port; - char *dnsserver_msg; + String dnsError; ///< DNS lookup error message time_t ttl; IpAddress src_addr; diff --git a/src/forward.cc b/src/forward.cc index 330a02cad2..6f0895726b 100644 --- a/src/forward.cc +++ b/src/forward.cc @@ -375,10 +375,10 @@ fwdNegotiateSSLWrapper(int fd, void *data) #endif static void -fwdConnectDoneWrapper(int server_fd, comm_err_t status, int xerrno, void *data) +fwdConnectDoneWrapper(int server_fd, const DnsLookupDetails &dns, comm_err_t status, int xerrno, void *data) { FwdState *fwd = (FwdState *) data; - fwd->connectDone(server_fd, status, xerrno); + fwd->connectDone(server_fd, dns, status, xerrno); } static void @@ -675,11 +675,13 @@ FwdState::initiateSSL() #endif void -FwdState::connectDone(int aServerFD, comm_err_t status, int xerrno) +FwdState::connectDone(int aServerFD, const DnsLookupDetails &dns, comm_err_t status, int xerrno) { FwdServer *fs = servers; assert(server_fd == aServerFD); + request->recordLookup(dns); + if (Config.onoff.log_ip_on_direct && status != COMM_ERR_DNS && fs->code == HIER_DIRECT) updateHierarchyInfo(); @@ -697,7 +699,7 @@ FwdState::connectDone(int aServerFD, comm_err_t status, int xerrno) ErrorState *anErr = errorCon(ERR_DNS_FAIL, HTTP_SERVICE_UNAVAILABLE, request); - anErr->dnsserver_msg = xstrdup(dns_error_message_safe()); + anErr->dnsError = dns.error; fail(anErr); @@ -778,6 +780,9 @@ FwdState::connectStart() assert(server_fd == -1); debugs(17, 3, "fwdConnectStart: " << url); + if (n_tries == 0) // first attempt + request->hier.first_conn_start = current_time; + if (fs->_peer) { ctimeout = fs->_peer->connect_timeout > 0 ? fs->_peer->connect_timeout : Config.Timeout.peer_connect; @@ -813,7 +818,7 @@ FwdState::connectStart() request->flags.auth = 1; comm_add_close_handler(fd, fwdServerClosedWrapper, this); updateHierarchyInfo(); - connectDone(fd, COMM_OK, 0); + connectDone(fd, DnsLookupDetails(), COMM_OK, 0); return; } /* Failure. Fall back on next path */ diff --git a/src/forward.h b/src/forward.h index c35dddef05..a1a737fb12 100644 --- a/src/forward.h +++ b/src/forward.h @@ -37,7 +37,7 @@ public: bool reforwardableStatus(http_status s); void serverClosed(int fd); void connectStart(); - void connectDone(int server_fd, comm_err_t status, int xerrno); + void connectDone(int server_fd, const DnsLookupDetails &dns, comm_err_t status, int xerrno); void connectTimeout(int fd); void initiateSSL(); void negotiateSSL(int fd); diff --git a/src/fqdncache.cc b/src/fqdncache.cc index 1e55d4fb52..ca534a7688 100644 --- a/src/fqdncache.cc +++ b/src/fqdncache.cc @@ -83,9 +83,6 @@ /// \ingroup FQDNCacheInternal #define FQDN_HIGH_WATER 95 -/// \ingroup FQDNCacheAPI -typedef struct _fqdncache_entry fqdncache_entry; - /** \ingroup FQDNCacheAPI * The data structure used for storing name-address mappings @@ -93,7 +90,8 @@ typedef struct _fqdncache_entry fqdncache_entry; * where structures of type fqdncache_entry whose most * interesting members are: */ -struct _fqdncache_entry { +class fqdncache_entry { +public: hash_link hash; /* must be first */ time_t lastref; time_t expires; @@ -111,6 +109,8 @@ struct _fqdncache_entry { unsigned int negcached:1; unsigned int fromhosts:1; } flags; + + int age() const; ///< time passed since request_time or -1 if unknown }; /// \ingroup FQDNCacheInternal @@ -134,7 +134,7 @@ static int fqdncacheParse(fqdncache_entry *, rfc1035_rr *, int, const char *erro #endif static void fqdncacheRelease(fqdncache_entry *); static fqdncache_entry *fqdncacheCreateEntry(const char *name); -static void fqdncacheCallback(fqdncache_entry *); +static void fqdncacheCallback(fqdncache_entry *, int wait); static fqdncache_entry *fqdncache_get(const char *); static FQDNH dummy_handler; static int fqdncacheExpiredEntry(const fqdncache_entry *); @@ -152,6 +152,13 @@ static long fqdncache_low = 180; /// \ingroup FQDNCacheInternal static long fqdncache_high = 200; +int +fqdncache_entry::age() const +{ + return request_time.tv_sec ? tvSubMsec(request_time, current_time) : -1; +} + + /** \ingroup FQDNCacheInternal * Removes the given fqdncache entry @@ -304,7 +311,7 @@ fqdncacheAddEntry(fqdncache_entry * f) * Walks down the pending list, calling handlers */ static void -fqdncacheCallback(fqdncache_entry * f) +fqdncacheCallback(fqdncache_entry * f, int wait) { FQDNH *callback; void *cbdata; @@ -320,8 +327,8 @@ fqdncacheCallback(fqdncache_entry * f) f->handler = NULL; if (cbdataReferenceValidDone(f->handlerData, &cbdata)) { - dns_error_message = f->error_message; - callback(f->name_count ? f->names[0] : NULL, cbdata); + const DnsLookupDetails details(f->error_message, wait); + callback(f->name_count ? f->names[0] : NULL, details, cbdata); } fqdncacheUnlockEntry(f); @@ -490,8 +497,8 @@ fqdncacheHandleReply(void *data, rfc1035_rr * answers, int na, const char *error fqdncache_entry *f; static_cast(data)->unwrap(&f); n = ++FqdncacheStats.replies; - statHistCount(&statCounter.dns.svc_time, - tvSubMsec(f->request_time, current_time)); + const int age = f->age(); + statHistCount(&statCounter.dns.svc_time, age); #if USE_DNSSERVERS fqdncacheParse(f, reply); @@ -502,7 +509,7 @@ fqdncacheHandleReply(void *data, rfc1035_rr * answers, int na, const char *error fqdncacheAddEntry(f); - fqdncacheCallback(f); + fqdncacheCallback(f, age); } /** @@ -528,8 +535,8 @@ fqdncache_nbgethostbyaddr(IpAddress &addr, FQDNH * handler, void *handlerData) if (name[0] == '\0') { debugs(35, 4, "fqdncache_nbgethostbyaddr: Invalid name!"); - dns_error_message = "Invalid hostname"; - handler(NULL, handlerData); + const DnsLookupDetails details("Invalid hostname", -1); // error, no lookup + handler(NULL, details, handlerData); return; } @@ -555,7 +562,7 @@ fqdncache_nbgethostbyaddr(IpAddress &addr, FQDNH * handler, void *handlerData) f->handlerData = cbdataReference(handlerData); - fqdncacheCallback(f); + fqdncacheCallback(f, -1); // no lookup return; } @@ -654,16 +661,16 @@ fqdncache_gethostbyaddr(IpAddress &addr, int flags) f = NULL; } else if (f->flags.negcached) { FqdncacheStats.negative_hits++; - dns_error_message = f->error_message; + // ignore f->error_message: the caller just checks FQDN cache presence return NULL; } else { FqdncacheStats.hits++; f->lastref = squid_curtime; - dns_error_message = f->error_message; + // ignore f->error_message: the caller just checks FQDN cache presence return f->names[0]; } - dns_error_message = NULL; + /* no entry [any more] */ FqdncacheStats.misses++; @@ -732,7 +739,7 @@ fqdnStats(StoreEntry * sentry) /// \ingroup FQDNCacheInternal static void -dummy_handler(const char *bufnotused, void *datanotused) +dummy_handler(const char *, const DnsLookupDetails &, void *) { return; } @@ -932,14 +939,3 @@ snmp_netFqdnFn(variable_list * Var, snint * ErrP) } #endif /*SQUID_SNMP */ - -/// XXX: a hack to work around the missing DNS error info -// see http://www.squid-cache.org/bugs/show_bug.cgi?id=2459 -const char * -dns_error_message_safe() -{ - if (dns_error_message) - return dns_error_message; - debugs(35,DBG_IMPORTANT, "Internal error: lost DNS error info"); - return "lost DNS error"; -} diff --git a/src/ftp.cc b/src/ftp.cc index 509755b76d..d0e1945f83 100644 --- a/src/ftp.cc +++ b/src/ftp.cc @@ -2787,10 +2787,11 @@ ftpReadPasv(FtpStateData * ftpState) } void -FtpStateData::ftpPasvCallback(int fd, comm_err_t status, int xerrno, void *data) +FtpStateData::ftpPasvCallback(int fd, const DnsLookupDetails &dns, comm_err_t status, int xerrno, void *data) { FtpStateData *ftpState = (FtpStateData *)data; debugs(9, 3, HERE); + ftpState->request->recordLookup(dns); if (status != COMM_OK) { debugs(9, 2, HERE << "Failed to connect. Retrying without PASV."); diff --git a/src/globals.h b/src/globals.h index 0da0a0e04b..dfd234bb1f 100644 --- a/src/globals.h +++ b/src/globals.h @@ -55,7 +55,6 @@ extern "C" //MOVED:structs.h extern SquidConfig2 Config2; extern char *ConfigFile; /* NULL */ extern char *IcpOpcodeStr[]; - extern const char *dns_error_message; /* NULL */ extern const char *log_tags[]; extern char tmp_error_buf[ERROR_BUF_SZ]; extern char ThisCache[RFC2181_MAXHOSTNAMELEN << 1]; diff --git a/src/http.cc b/src/http.cc index 49d25fd350..b2d02b9a3d 100644 --- a/src/http.cc +++ b/src/http.cc @@ -87,6 +87,10 @@ HttpStateData::HttpStateData(FwdState *theFwdState) : AsyncJob("HttpStateData"), readBuf->init(); orig_request = HTTPMSGLOCK(fwd->request); + // reset peer response time stats for %hier.peer_http_request_sent.tv_sec = 0; + orig_request->hier.peer_http_request_sent.tv_usec = 0; + if (fwd->servers) _peer = fwd->servers->_peer; /* might be NULL */ @@ -732,6 +736,8 @@ HttpStateData::processReplyHeader() * Parse the header and remove all referenced headers */ + orig_request->hier.peer_reply_status = newrep->sline.status; + ctx_exit(ctx); } @@ -1064,6 +1070,11 @@ HttpStateData::readReply(const CommIoCbParams &io) clen >>= 1; IOStats.Http.read_hist[bin]++; + + // update peer response time stats (%hier.peer_http_request_sent; + orig_request->hier.peer_response_time = + sent.tv_sec ? tvSubMsec(sent, current_time) : -1; } /** \par @@ -1410,6 +1421,8 @@ HttpStateData::sendComplete(const CommIoCbParams &io) commSetTimeout(fd, Config.Timeout.read, timeoutCall); flags.request_sent = 1; + + orig_request->hier.peer_http_request_sent = current_time; } // Close the HTTP server connection. Used by serverComplete(). diff --git a/src/icmp/net_db.cc b/src/icmp/net_db.cc index b9a6d3b7e9..84c771799b 100644 --- a/src/icmp/net_db.cc +++ b/src/icmp/net_db.cc @@ -271,7 +271,7 @@ netdbAdd(IpAddress &addr) } static void -netdbSendPing(const ipcache_addrs * ia, void *data) +netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data) { IpAddress addr; char *hostname = NULL; diff --git a/src/ident/Ident.cc b/src/ident/Ident.cc index 90014cc1f6..31e929464d 100644 --- a/src/ident/Ident.cc +++ b/src/ident/Ident.cc @@ -120,7 +120,7 @@ Ident::Timeout(int fd, void *data) } static void -Ident::ConnectDone(int fd, comm_err_t status, int xerrno, void *data) +Ident::ConnectDone(int fd, const DnsLookupDetails &, comm_err_t status, int xerrno, void *data) { IdentStateData *state = (IdentStateData *)data; IdentClient *c; diff --git a/src/ipcache.cc b/src/ipcache.cc index e41484e04d..a7f81e7010 100644 --- a/src/ipcache.cc +++ b/src/ipcache.cc @@ -77,9 +77,6 @@ * the ipcache_high threshold. */ -/// \ingroup IPCacheAPI -typedef struct _ipcache_entry ipcache_entry; - /** \ingroup IPCacheAPI * @@ -88,7 +85,8 @@ typedef struct _ipcache_entry ipcache_entry; * where structures of type ipcache_entry whose most * interesting members are: */ -struct _ipcache_entry { +class ipcache_entry { +public: hash_link hash; /* must be first */ time_t lastref; time_t expires; @@ -108,6 +106,8 @@ struct _ipcache_entry { unsigned int negcached:1; unsigned int fromhosts:1; } flags; + + int age() const; ///< time passed since request_time or -1 if unknown }; /// \ingroup IPCacheInternal @@ -161,6 +161,14 @@ static long ipcache_high = 200; extern int _dns_ttl_; #endif +int +ipcache_entry::age() const +{ + return request_time.tv_sec ? tvSubMsec(request_time, current_time) : -1; +} + + + /** \ingroup IPCacheInternal * @@ -324,7 +332,7 @@ ipcacheAddEntry(ipcache_entry * i) * walks down the pending list, calling handlers */ static void -ipcacheCallback(ipcache_entry * i) +ipcacheCallback(ipcache_entry *i, int wait) { IPH *callback = i->handler; void *cbdata = NULL; @@ -340,8 +348,8 @@ ipcacheCallback(ipcache_entry * i) i->handler = NULL; if (cbdataReferenceValidDone(i->handlerData, &cbdata)) { - dns_error_message = i->error_message; - callback(i->addrs.count ? &i->addrs : NULL, cbdata); + const DnsLookupDetails details(i->error_message, wait); + callback((i->addrs.count ? &i->addrs : NULL), details, cbdata); } ipcacheUnlockEntry(i); @@ -646,8 +654,9 @@ ipcacheHandleReply(void *data, rfc1035_rr * answers, int na, const char *error_m ipcache_entry *i; static_cast(data)->unwrap(&i); IpcacheStats.replies++; - statHistCount(&statCounter.dns.svc_time, - tvSubMsec(i->request_time, current_time)); + const int age = i->age(); + statHistCount(&statCounter.dns.svc_time, age); + #if USE_DNSSERVERS done = ipcacheParse(i, reply); @@ -661,7 +670,7 @@ ipcacheHandleReply(void *data, rfc1035_rr * answers, int na, const char *error_m { ipcacheAddEntry(i); - ipcacheCallback(i); + ipcacheCallback(i, age); } } @@ -694,16 +703,16 @@ ipcache_nbgethostbyname(const char *name, IPH * handler, void *handlerData) if (name == NULL || name[0] == '\0') { debugs(14, 4, "ipcache_nbgethostbyname: Invalid name!"); IpcacheStats.invalid++; - dns_error_message = "Invalid hostname"; - handler(NULL, handlerData); + const DnsLookupDetails details("Invalid hostname", -1); // error, no lookup + handler(NULL, details, handlerData); return; } if ((addrs = ipcacheCheckNumeric(name))) { debugs(14, 4, "ipcache_nbgethostbyname: BYPASS for '" << name << "' (already numeric)"); - dns_error_message = NULL; IpcacheStats.numeric_hits++; - handler(addrs, handlerData); + const DnsLookupDetails details(NULL, -1); // no error, no lookup + handler(addrs, details, handlerData); return; } @@ -729,7 +738,7 @@ ipcache_nbgethostbyname(const char *name, IPH * handler, void *handlerData) i->handlerData = cbdataReference(handlerData); - ipcacheCallback(i); + ipcacheCallback(i, -1); // no lookup return; } @@ -823,16 +832,16 @@ ipcache_gethostbyname(const char *name, int flags) i = NULL; } else if (i->flags.negcached) { IpcacheStats.negative_hits++; - dns_error_message = i->error_message; + // ignore i->error_message: the caller just checks IP cache presence return NULL; } else { IpcacheStats.hits++; i->lastref = squid_curtime; - dns_error_message = i->error_message; + // ignore i->error_message: the caller just checks IP cache presence return &i->addrs; } - dns_error_message = NULL; + /* no entry [any more] */ if ((addrs = ipcacheCheckNumeric(name))) { IpcacheStats.numeric_hits++; @@ -1064,7 +1073,7 @@ ipcacheMergeIPLists(const IpAddress *aaddrs, const int alen, /// \ingroup IPCacheInternal /// Callback. static void -ipcacheHandleCnameRecurse(const ipcache_addrs *addrs, void *cbdata) +ipcacheHandleCnameRecurse(const ipcache_addrs *addrs, const DnsLookupDetails &, void *cbdata) { #if DNS_CNAME ipcache_entry *i = NULL; @@ -1159,7 +1168,7 @@ ipcacheHandleCnameRecurse(const ipcache_addrs *addrs, void *cbdata) /* finish the lookup we were doing on parent when we got side-tracked for CNAME loop */ if (i->cname_wait == 0) { ipcacheAddEntry(i); - ipcacheCallback(i); + ipcacheCallback(i, i->age()); // age since i creation, includes CNAMEs } // else still more CNAME to be found. #endif /* DNS_CNAME */ diff --git a/src/main.cc b/src/main.cc index 2c785168d5..00bb217c56 100644 --- a/src/main.cc +++ b/src/main.cc @@ -34,6 +34,9 @@ #include "squid.h" #include "AccessLogEntry.h" +#if ICAP_CLIENT +#include "adaptation/icap/icap_log.h" +#endif #include "auth/Gadgets.h" #include "ConfigParser.h" #include "errorpage.h" @@ -695,6 +698,9 @@ mainReconfigureStart(void) storeDirCloseSwapLogs(); storeLogClose(); accessLogClose(); +#if ICAP_CLIENT + icapLogClose(); +#endif useragentLogClose(); refererCloseLog(); @@ -726,6 +732,9 @@ mainReconfigureFinish(void *) parseEtcHosts(); errorInitialize(); /* reload error pages */ accessLogInit(); +#if ICAP_CLIENT + icapLogOpen(); +#endif storeLogOpen(); useragentOpenLog(); refererOpenLog(); @@ -790,6 +799,9 @@ mainRotate(void) accessLogRotate(); /* access.log */ useragentRotateLog(); /* useragent.log */ refererRotateLog(); /* referer.log */ +#if ICAP_CLIENT + icapLogRotate(); /*icap.log*/ +#endif #if WIP_FWD_LOG fwdLogRotate(); @@ -951,6 +963,10 @@ mainInitialize(void) accessLogInit(); +#if ICAP_CLIENT + icapLogOpen(); +#endif + #if USE_IDENT Ident::Init(); #endif diff --git a/src/multicast.cc b/src/multicast.cc index 385f059729..44b62aa7cb 100644 --- a/src/multicast.cc +++ b/src/multicast.cc @@ -50,7 +50,7 @@ mcastSetTtl(int fd, int mcast_ttl) } void -mcastJoinGroups(const ipcache_addrs * ia, void *datanotused) +mcastJoinGroups(const ipcache_addrs *ia, const DnsLookupDetails &, void *datanotused) { #ifdef IP_MULTICAST_TTL int fd = theInIcpConnection; diff --git a/src/neighbors.cc b/src/neighbors.cc index 3a190fbd9b..00ef0165c7 100644 --- a/src/neighbors.cc +++ b/src/neighbors.cc @@ -1230,7 +1230,7 @@ peerNoteDigestGone(peer * p) } static void -peerDNSConfigure(const ipcache_addrs * ia, void *data) +peerDNSConfigure(const ipcache_addrs *ia, const DnsLookupDetails &, void *data) { peer *p = (peer *)data; @@ -1399,7 +1399,7 @@ peerProbeConnect(peer * p) } static void -peerProbeConnectDone(int fd, comm_err_t status, int xerrno, void *data) +peerProbeConnectDone(int fd, const DnsLookupDetails &, comm_err_t status, int xerrno, void *data) { peer *p = (peer*)data; diff --git a/src/protos.h b/src/protos.h index 90fa2ea355..8d63ce34fb 100644 --- a/src/protos.h +++ b/src/protos.h @@ -127,8 +127,6 @@ SQUIDCEXTERN void disk_init(void); SQUIDCEXTERN void dnsShutdown(void); SQUIDCEXTERN void dnsInit(void); SQUIDCEXTERN void dnsSubmit(const char *lookup, HLPCB * callback, void *data); -/// XXX: a temporary hack to work around the missing DNS error info -const char *dns_error_message_safe(); /* dns_internal.c */ SQUIDCEXTERN void idnsInit(void); diff --git a/src/send-announce.cc b/src/send-announce.cc index 6525c81640..46f07114b9 100644 --- a/src/send-announce.cc +++ b/src/send-announce.cc @@ -55,7 +55,7 @@ start_announce(void *datanotused) } static void -send_announce(const ipcache_addrs * ia, void *junk) +send_announce(const ipcache_addrs *ia, const DnsLookupDetails &, void *junk) { LOCAL_ARRAY(char, tbuf, 256); LOCAL_ARRAY(char, sndbuf, BUFSIZ); diff --git a/src/structs.h b/src/structs.h index ab2d305706..198f35eb82 100644 --- a/src/structs.h +++ b/src/structs.h @@ -183,6 +183,7 @@ struct SquidConfig { } Timeout; size_t maxRequestHeaderSize; int64_t maxRequestBodySize; + int64_t maxChunkedRequestBodySize; size_t maxReplyHeaderSize; acl_size_t *ReplyBodySize; @@ -264,6 +265,10 @@ struct SquidConfig { customlog *accesslogs; +#if ICAP_CLIENT + customlog *icaplogs; +#endif + int rotateNumber; } Log; char *adminEmail; @@ -479,6 +484,9 @@ struct SquidConfig { acl_access *followXFF; #endif /* FOLLOW_X_FORWARDED_FOR */ +#if ICAP_CLIENT + acl_access* icap; +#endif } accessList; acl_deny_info_list *denyInfoList; authConfig authConfiguration; diff --git a/src/tunnel.cc b/src/tunnel.cc index cc6b813edd..a6bab679f4 100644 --- a/src/tunnel.cc +++ b/src/tunnel.cc @@ -553,12 +553,14 @@ tunnelErrorComplete(int fdnotused, void *data, size_t sizenotused) static void -tunnelConnectDone(int fdnotused, comm_err_t status, int xerrno, void *data) +tunnelConnectDone(int fdnotused, const DnsLookupDetails &dns, comm_err_t status, int xerrno, void *data) { TunnelStateData *tunnelState = (TunnelStateData *)data; HttpRequest *request = tunnelState->request; ErrorState *err = NULL; + request->recordLookup(dns); + if (tunnelState->servers->_peer) hierarchyNote(&tunnelState->request->hier, tunnelState->servers->code, tunnelState->servers->_peer->host); @@ -573,7 +575,7 @@ tunnelConnectDone(int fdnotused, comm_err_t status, int xerrno, void *data) debugs(26, 4, "tunnelConnect: Unknown host: " << tunnelState->host); err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND, request); *tunnelState->status_ptr = HTTP_NOT_FOUND; - err->dnsserver_msg = xstrdup(dns_error_message_safe()); + err->dnsError = dns.error; err->callback = tunnelErrorComplete; err->callback_data = tunnelState; errorSend(tunnelState->client.fd(), err); diff --git a/src/typedefs.h b/src/typedefs.h index 474864f278..8925243713 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -199,9 +199,10 @@ typedef void DCCB(int, int errflag, void *data); /* disk close CB */ typedef void DUCB(int errflag, void *data); /* disk unlink CB */ typedef void DTCB(int errflag, void *data); /* disk trunc CB */ -typedef void FQDNH(const char *, void *); +class DnsLookupDetails; +typedef void FQDNH(const char *, const DnsLookupDetails &details, void *); typedef void IDCB(const char *ident, void *data); -typedef void IPH(const ipcache_addrs *, void *); +typedef void IPH(const ipcache_addrs *, const DnsLookupDetails &details, void *); typedef void IRCB(struct peer *, peer_t, protocol_t, void *, void *data); class FwdServer;