From: Amos Jeffries Date: Sun, 12 Dec 2010 05:30:58 +0000 (-0700) Subject: SourceLayout: cleanup the various log line formatting code X-Git-Tag: take00~48 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=20efa1c285cf589c5e289fd4f07bf41ef3564fd6;p=thirdparty%2Fsquid.git SourceLayout: cleanup the various log line formatting code Adds: * namespace Log::Format for log display functionality. Each line formater is a global function inside here. The log format enum is also in here along with the display encoding 'gadget' functions. * namespace Time in SquidTime.h for the related time string display functions. Unified the various log pretty-print httpd-style time functions into Time::FormatHttpd(time_t). ** care has been taken to preserve the local-static optimization found in accessLogTime() to prevent wasted cycles re-printing the same time value more than once per second. NP: the similar but timezone-missing format is now Time::FormatStrf() with the same optimization applied to speed up its callers. * namespace Math:: to avoid symbol clash with global function Log() and namespace Log. * support for the Apache "combined" log format. Was documented earlier as being available but not actually present. Obsoletes: * forward_log directive and associated experimental code. If needed we can easily add another special format to dump the details. FWIW they are all available in the squid format anyway (timestamp, squid status, source peer). The documented action of dumping every forwarding attempt was not working. * referer_log and useragent_log directives and matching ./configure options. ** shuffled into access_log formats "referrer" and "useragent" for more flexibility with less directives. * emulate_httpd_log replaced with Apache "common" format. * the "auto" pseudo-format becomes obsolete with emulat_httpd_log. default is now "squid" format in all situations. Code Shuffles: * moved the logformat directive parsing into LogConfig object methods. * shuffled the logformat parsing and token code into src/log/Tokens.h|cc ** this is purely to break it out of access_log.cc. namespace and scoping needs some work. --- diff --git a/configure.ac b/configure.ac index b14ad08ccc..d029f6507d 100644 --- a/configure.ac +++ b/configure.ac @@ -954,28 +954,6 @@ dnl fi dnl ]) -AC_ARG_ENABLE(useragent-log, - AS_HELP_STRING([--enable-useragent-log], - [Enable logging of User-Agent header]), [ -SQUID_YESNO([$enableval], - [unrecognized argument to --enable-useragent-log: $enableval]) - enable_useragent_log=$enableval -]) -SQUID_DEFINE_BOOL(USE_USERAGENT_LOG,${enable_useragent_log:=no}, - [If you want to log User-Agent request header values, define this.]) -AC_MSG_NOTICE([User-Agent logging enabled: $enable_useragent_log]) - -AC_ARG_ENABLE(referer-log, - AS_HELP_STRING([--enable-referer-log],[Enable logging of Referer header]), [ -SQUID_YESNO([$enableval], - [unrecognized argument to --enable-referer-log: $enableval]) -]) -SQUID_DEFINE_BOOL(USE_REFERER_LOG,${enable_referer_log:=no}, - [If you want to log Referer request header values, define this. - By default, they are written to referer.log in the Squid logdir. - This feature is deprecated in favour of custom log formats]) -AC_MSG_NOTICE([Referer logging enabled: $enable_referer_log]) - AC_ARG_ENABLE(wccp, AS_HELP_STRING([--disable-wccp],[Disable Web Cache Coordination Protocol]), [ SQUID_YESNO([$enableval],[unrecognized argument to --disable-wccp: $enableval]) diff --git a/doc/debug-sections.txt b/doc/debug-sections.txt index 1289116ac0..c923a962ce 100644 --- a/doc/debug-sections.txt +++ b/doc/debug-sections.txt @@ -1,14 +1,10 @@ -section -- CGI Cache Manager section -- External DISKD process implementation. -section -- Refcount allocator section -- Unlink Daemon -section -- WWW Client section 00 Announcement Server section 00 Client Database section 00 DNS Resolver Daemon section 00 Debug Routines -section 00 Hash Tables section 00 UFS Store Dump Tool section 01 Main Loop section 01 Startup and Main Loop @@ -70,8 +66,6 @@ section 38 Network Measurement Database section 39 Cache Array Routing Protocol section 39 Peer source hash based selection section 39 Peer user hash based selection -section 40 Referer Logging -section 40 User-Agent Logging section 41 Event Processing section 42 ICMP Pinger program section 43 AIOPS @@ -79,6 +73,14 @@ section 43 Windows AIOPS section 44 Peer Selection Algorithm section 45 Callback Data Registry section 46 Access Log +section 46 Access Log - Apache combined format +section 46 Access Log - Apache common format +section 46 Access Log - Squid Custom format +section 46 Access Log - Squid ICAP Logging +section 46 Access Log - Squid format +section 46 Access Log - Squid referer format +section 46 Access Log - Squid useragent format +section 46 Access Log Format Tokens section 47 Store COSS Directory Routines section 47 Store Directory Routines section 48 Persistent Connections @@ -88,7 +90,6 @@ section 50 Log file handling section 51 Filedescriptor Functions section 52 URN Parsing section 53 AS Number handling -section 53 Radix Tree data structure implementation section 54 Interprocess Communication section 54 Windows Interprocess Communication section 55 HTTP Header @@ -99,7 +100,6 @@ section 59 auto-growing Memory Buffer with printf section 60 Packer: A uniform interface to store-like modules section 61 Redirector section 62 Generic Histogram -section 63 Low Level Memory Pool Management section 64 HTTP Range Header section 65 HTTP Cache Control Header section 66 HTTP Header Tools @@ -129,7 +129,6 @@ section 82 External ACL section 83 SSL accelerator support section 84 Helper process maintenance section 85 Client-side Request Routines -section 86 ESI Expressions section 86 ESI processing section 87 Client-side Stream routines. section 88 Client-side Reply Routines diff --git a/doc/release-notes/release-3.2.sgml b/doc/release-notes/release-3.2.sgml index 5216ae3ba6..66a6af233d 100644 --- a/doc/release-notes/release-3.2.sgml +++ b/doc/release-notes/release-3.2.sgml @@ -287,6 +287,8 @@ Most user-facing changes are reflected in squid.conf (see below). logging a single cache.log at relatively high debug levels on a high-traffic system. Or one which is required to store a long period of access.log and needs to conserve disk space. +

The referer_log and useragent_log directives have been converted to built-in log formats. + These logs are now created using an access_log line with the format "referrer" or "useragent". Client Bandwidth Limits

In mobile environments, Squid may need to limit Squid-to-client bandwidth @@ -448,6 +450,8 @@ This section gives a thorough account of those changes in three categories: New installs, or installs with no logs configured explicitly will use this module by default.

New tcp module to send each log line as text data to a TCP receiver.

New udp module to send each log line as text data to a UDP receiver. +

New format referrer to log with the format prevously used by referer_log directive. +

New format useragent to log with the format prevously used by useragent_log directive. acl random

New type random. Pseudo-randomly match requests based on a configured probability. @@ -535,6 +539,12 @@ This section gives a thorough account of those changes in three categories: Removed tags

+ emulate_httpd_log +

Replaced by common format option on an access_log directive. + + forward_log +

Obsolete. + ftp_list_width

Obsolete. @@ -544,9 +554,14 @@ This section gives a thorough account of those changes in three categories: log_fqdn

Obsolete. Replaced by automatic detection of the %>A logformat tag. + referer_log +

Replaced by the referrer format option on an access_log directive. + url_rewrite_concurrency

Replaced by url_rewrite_children ... concurrency=N option. + useragent_log +

Replaced by the useragent format option on an access_log directive. @@ -647,6 +662,12 @@ This section gives an account of those changes in three categories: --enable-auth-ntlm-helpers

replaced by --enable-auth-ntlm. + --enable-referer-log +

Obsolete. + + --enable-useragent-log +

Obsolete. + diff --git a/lib/rfc1123.c b/lib/rfc1123.c index 7d55737437..51accddc3c 100644 --- a/lib/rfc1123.c +++ b/lib/rfc1123.c @@ -246,48 +246,6 @@ mkrfc1123(time_t t) return buf; } -const char * -mkhttpdlogtime(const time_t * t) -{ - static char buf[128]; - - struct tm *gmt = gmtime(t); - -#if !USE_GMT - int gmt_min, gmt_hour, gmt_yday, day_offset; - size_t len; - struct tm *lt; - int min_offset; - - /* localtime & gmtime may use the same static data */ - gmt_min = gmt->tm_min; - gmt_hour = gmt->tm_hour; - gmt_yday = gmt->tm_yday; - - lt = localtime(t); - - day_offset = lt->tm_yday - gmt_yday; - /* wrap round on end of year */ - if (day_offset > 1) - day_offset = -1; - else if (day_offset < -1) - day_offset = 1; - - min_offset = day_offset * 1440 + (lt->tm_hour - gmt_hour) * 60 - + (lt->tm_min - gmt_min); - - len = strftime(buf, 127 - 5, "%d/%b/%Y:%H:%M:%S ", lt); - snprintf(buf + len, 128 - len, "%+03d%02d", - (min_offset / 60) % 24, - min_offset % 60); -#else /* USE_GMT */ - buf[0] = '\0'; - strftime(buf, 127, "%d/%b/%Y:%H:%M:%S -000", gmt); -#endif /* USE_GMT */ - - return buf; -} - #if 0 int main() diff --git a/src/Makefile.am b/src/Makefile.am index 1402f4f106..1d7345666c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -419,7 +419,6 @@ squid_SOURCES = \ PingData.h \ protos.h \ redirect.cc \ - referer.cc \ refresh.cc \ RemovalPolicy.cc \ RemovalPolicy.h \ @@ -488,7 +487,6 @@ squid_SOURCES = \ URLScheme.cc \ URLScheme.h \ urn.cc \ - useragent.cc \ wccp.cc \ wccp2.cc \ whois.cc \ @@ -1234,7 +1232,6 @@ tests_testCacheManager_SOURCES = \ peer_sourcehash.cc \ peer_userhash.cc \ redirect.cc \ - referer.cc \ refresh.cc \ RemovalPolicy.cc \ Server.cc \ @@ -1272,7 +1269,6 @@ tests_testCacheManager_SOURCES = \ url.cc \ URLScheme.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ @@ -1439,7 +1435,6 @@ tests_testEvent_SOURCES = \ peer_sourcehash.cc \ peer_userhash.cc \ redirect.cc \ - referer.cc \ refresh.cc \ Server.cc \ $(SNMP_SOURCE) \ @@ -1476,7 +1471,6 @@ tests_testEvent_SOURCES = \ url.cc \ URLScheme.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ @@ -1602,7 +1596,6 @@ tests_testEventLoop_SOURCES = \ peer_sourcehash.cc \ peer_userhash.cc \ redirect.cc \ - referer.cc \ refresh.cc \ Server.cc \ $(SNMP_SOURCE) \ @@ -1639,7 +1632,6 @@ tests_testEventLoop_SOURCES = \ url.cc \ URLScheme.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ @@ -1755,7 +1747,6 @@ tests_test_http_range_SOURCES = \ peer_userhash.cc \ pconn.cc \ redirect.cc \ - referer.cc \ refresh.cc \ RemovalPolicy.cc \ Server.cc \ @@ -1795,7 +1786,6 @@ tests_test_http_range_SOURCES = \ url.cc \ URLScheme.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ @@ -1922,7 +1912,6 @@ tests_testHttpRequest_SOURCES = \ peer_sourcehash.cc \ peer_userhash.cc \ redirect.cc \ - referer.cc \ refresh.cc \ RemovalPolicy.cc \ Server.cc \ @@ -1960,7 +1949,6 @@ tests_testHttpRequest_SOURCES = \ url.cc \ URLScheme.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ @@ -2340,7 +2328,6 @@ tests_testURL_SOURCES = \ peer_sourcehash.cc \ peer_userhash.cc \ redirect.cc \ - referer.cc \ refresh.cc \ Server.cc \ $(SNMP_SOURCE) \ @@ -2375,7 +2362,6 @@ tests_testURL_SOURCES = \ tunnel.cc \ SwapDir.cc \ urn.cc \ - useragent.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ diff --git a/src/SquidTime.h b/src/SquidTime.h index 47bd9533f1..46a84a47c3 100644 --- a/src/SquidTime.h +++ b/src/SquidTime.h @@ -58,4 +58,25 @@ public: virtual void tick(); }; +namespace Time +{ + +/** Display time as a formatted human-readable string. + * Time syntax is + * "YYYY/MM/DD hh:mm:ss" + * + * Output is only valid until next call to this function. + */ +const char *FormatStrf(time_t t); + +/** Display time as a formatted human-readable string. + * Time string syntax used is that of Apache httpd. + * "DD/MMM/YYYY:hh:mm:ss zzzz" + * + * Output is only valid until next call to this function. + */ +const char *FormatHttpd(time_t t); + +} // namespace Time + #endif /* SQUID_TIME_H */ diff --git a/src/StatHist.cc b/src/StatHist.cc index 26092e7e11..eaac62c53c 100644 --- a/src/StatHist.cc +++ b/src/StatHist.cc @@ -58,9 +58,12 @@ static StatHistBinDumper statHistBinDumper; * HP-UX and GCC (2.8?) give strange errors when these simple * functions are static. */ +namespace Math +{ static hbase_f Log; static hbase_f Exp; static hbase_f Null; +}; #endif /* low level init, higher level functions has less params */ @@ -278,7 +281,7 @@ statHistDump(const StatHist * H, StoreEntry * sentry, StatHistBinDumper * bd) static #endif double -Log(double x) +Math::Log(double x) { assert((x + 1.0) >= 0.0); return log(x + 1.0); @@ -288,7 +291,7 @@ Log(double x) static #endif double -Exp(double x) +Math::Exp(double x) { return exp(x) - 1.0; } @@ -296,7 +299,7 @@ Exp(double x) void statHistLogInit(StatHist * H, int capacity, double min, double max) { - statHistInit(H, capacity, Log, Exp, min, max); + statHistInit(H, capacity, Math::Log, Math::Exp, min, max); } /* linear histogram for enums */ @@ -305,7 +308,7 @@ statHistLogInit(StatHist * H, int capacity, double min, double max) static #endif double -Null(double x) +Math::Null(double x) { return x; } @@ -313,7 +316,7 @@ Null(double x) void statHistEnumInit(StatHist * H, int last_enum) { - statHistInit(H, last_enum + 3, Null, Null, (double) -1, (double) (last_enum + 1 + 1)); + statHistInit(H, last_enum + 3, Math::Null, Math::Null, (double) -1, (double) (last_enum + 1 + 1)); } void @@ -327,7 +330,7 @@ statHistEnumDumper(StoreEntry * sentry, int idx, double val, double size, int co void statHistIntInit(StatHist * H, int n) { - statHistInit(H, n, Null, Null, (double) 0, (double) n - 1); + statHistInit(H, n, Math::Null, Math::Null, (double) 0, (double) n - 1); } void diff --git a/src/adaptation/icap/icap_log.cc b/src/adaptation/icap/icap_log.cc index b9f85e6073..3371c678ee 100644 --- a/src/adaptation/icap/icap_log.cc +++ b/src/adaptation/icap/icap_log.cc @@ -2,6 +2,7 @@ #include "icap_log.h" #include "AccessLogEntry.h" #include "log/File.h" +#include "log/Formats.h" int IcapLogfileStatus = LOG_DISABLE; @@ -11,11 +12,11 @@ icapLogOpen() customlog *log; for (log = Config.Log.icaplogs; log; log = log->next) { - if (log->type == CLF_NONE) + if (log->type == Log::Format::CLF_NONE) continue; - if (log->type == CLF_AUTO) - log->type = CLF_ICAP_SQUID; + if (log->type == Log::Format::CLF_AUTO) + log->type = Log::Format::CLF_ICAP_SQUID; log->logfile = logfileOpen(log->filename, MAX_URL << 1, 1); diff --git a/src/cache_cf.cc b/src/cache_cf.cc index 12fd981e55..82c9317883 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -133,13 +133,9 @@ static const char *const B_GBYTES_STR = "GB"; static const char *const list_sep = ", \t\n\r"; -static void parse_logformat(logformat ** logformat_definitions); static void parse_access_log(customlog ** customlog_definitions); static int check_null_access_log(customlog *customlog_definitions); - -static void dump_logformat(StoreEntry * entry, const char *name, logformat * definitions); static void dump_access_log(StoreEntry * entry, const char *name, customlog * definitions); -static void free_logformat(logformat ** definitions); static void free_access_log(customlog ** definitions); static void update_maxobjsize(void); @@ -3975,36 +3971,6 @@ strtokFile(void) } #include "AccessLogEntry.h" -/* TODO: split out parsing somehow ...*/ -static void -parse_logformat(logformat ** logformat_definitions) -{ - logformat *nlf; - char *name, *def; - - if ((name = strtok(NULL, w_space)) == NULL) - self_destruct(); - - if ((def = strtok(NULL, "\r\n")) == NULL) { - self_destruct(); - return; - } - - debugs(3, 2, "Logformat for '" << name << "' is '" << def << "'"); - - nlf = (logformat *)xcalloc(1, sizeof(logformat)); - - nlf->name = xstrdup(name); - - if (!accessLogParseLogFormat(&nlf->format, def)) { - self_destruct(); - return; - } - - nlf->next = *logformat_definitions; - - *logformat_definitions = nlf; -} static void parse_access_log(customlog ** logs) @@ -4021,19 +3987,19 @@ parse_access_log(customlog ** logs) } if (strcmp(filename, "none") == 0) { - cl->type = CLF_NONE; + cl->type = Log::Format::CLF_NONE; goto done; } if ((logdef_name = strtok(NULL, w_space)) == NULL) - logdef_name = "auto"; + logdef_name = "squid"; debugs(3, 9, "Log definition name '" << logdef_name << "' file '" << filename << "'"); cl->filename = xstrdup(filename); /* look for the definition pointer corresponding to this name */ - lf = Config.Log.logformats; + lf = Log::TheConfig.logformats; while (lf != NULL) { debugs(3, 9, "Comparing against '" << lf->name << "'"); @@ -4045,18 +4011,25 @@ parse_access_log(customlog ** logs) } if (lf != NULL) { - cl->type = CLF_CUSTOM; + cl->type = Log::Format::CLF_CUSTOM; cl->logFormat = lf; } else if (strcmp(logdef_name, "auto") == 0) { - cl->type = CLF_AUTO; + debugs(0,0, "WARNING: Log format 'auto' no longer exists. Using 'squid' instead."); + cl->type = Log::Format::CLF_SQUID; } else if (strcmp(logdef_name, "squid") == 0) { - cl->type = CLF_SQUID; + cl->type = Log::Format::CLF_SQUID; } else if (strcmp(logdef_name, "common") == 0) { - cl->type = CLF_COMMON; + cl->type = Log::Format::CLF_COMMON; + } else if (strcmp(logdef_name, "combined") == 0) { + cl->type = Log::Format::CLF_COMBINED; #if ICAP_CLIENT } else if (strcmp(logdef_name, "icap_squid") == 0) { - cl->type = CLF_ICAP_SQUID; + cl->type = Log::Format::CLF_ICAP_SQUID; #endif + } else if (strcmp(logdef_name, "useragent") == 0) { + cl->type = Log::Format::CLF_USERAGENT; + } else if (strcmp(logdef_name, "referrer") == 0) { + cl->type = Log::Format::CLF_REFERER; } else { debugs(3, 0, "Log format '" << logdef_name << "' is not defined"); self_destruct(); @@ -4078,12 +4051,6 @@ check_null_access_log(customlog *customlog_definitions) return customlog_definitions == NULL; } -static void -dump_logformat(StoreEntry * entry, const char *name, logformat * definitions) -{ - accessLogDumpLogFormat(entry, name, definitions); -} - static void dump_access_log(StoreEntry * entry, const char *name, customlog * logs) { @@ -4094,36 +4061,40 @@ dump_access_log(StoreEntry * entry, const char *name, customlog * logs) switch (log->type) { - case CLF_CUSTOM: + case Log::Format::CLF_CUSTOM: storeAppendPrintf(entry, "%s %s", log->filename, log->logFormat->name); break; - case CLF_NONE: + case Log::Format::CLF_NONE: storeAppendPrintf(entry, "none"); break; - case CLF_SQUID: + case Log::Format::CLF_SQUID: storeAppendPrintf(entry, "%s squid", log->filename); break; - case CLF_COMMON: - storeAppendPrintf(entry, "%s squid", log->filename); + case Log::Format::CLF_COMBINED: + storeAppendPrintf(entry, "%s combined", log->filename); + break; + + case Log::Format::CLF_COMMON: + storeAppendPrintf(entry, "%s common", log->filename); break; + #if ICAP_CLIENT - case CLF_ICAP_SQUID: + case Log::Format::CLF_ICAP_SQUID: storeAppendPrintf(entry, "%s icap_squid", log->filename); break; #endif - case CLF_AUTO: - - if (log->aclList) - storeAppendPrintf(entry, "%s auto", log->filename); - else - storeAppendPrintf(entry, "%s", log->filename); + case Log::Format::CLF_USERAGENT: + storeAppendPrintf(entry, "%s useragent", log->filename); + break; + case Log::Format::CLF_REFERER: + storeAppendPrintf(entry, "%s referrer", log->filename); break; - case CLF_UNKNOWN: + case Log::Format::CLF_UNKNOWN: break; } @@ -4134,18 +4105,6 @@ dump_access_log(StoreEntry * entry, const char *name, customlog * logs) } } -static void -free_logformat(logformat ** definitions) -{ - while (*definitions) { - logformat *format = *definitions; - *definitions = format->next; - safe_free(format->name); - accessLogFreeLogFormat(&format->format); - xfree(format); - } -} - static void free_access_log(customlog ** definitions) { @@ -4154,7 +4113,7 @@ free_access_log(customlog ** definitions) *definitions = log->next; log->logFormat = NULL; - log->type = CLF_UNKNOWN; + log->type = Log::Format::CLF_UNKNOWN; if (log->aclList) aclDestroyAclList(&log->aclList); diff --git a/src/cf.data.pre b/src/cf.data.pre index 3aaa0b68ae..8f5e1f383f 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -2817,7 +2817,7 @@ COMMENT_END NAME: logformat TYPE: logformat -LOC: Config.Log.logformats +LOC: Log::TheConfig DEFAULT: none DOC_START Usage: @@ -2962,10 +2962,18 @@ DOC_START The default formats available (which do not need re-defining) are: -logformat squid %ts.%03tu %6tr %>a %Ss/%03>Hs %a %Ss/%03>Hs %h] [%a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh +logformat squid %ts.%03tu %6tr %>a %Ss/%03>Hs %a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %a %ui %un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh +logformat referrer %ts.%03tu %>a %{Referer}>h %ru +logformat useragent %>a [%tl] "%{User-Agent}>h" + + When the log_mime_hdrs directive is set to ON. The squid, common and combined + formats have a safely encoded copy of the mime headers appended to each line + within a pair of brackets. + + The common and combined formats are not quite true to the Apache definition. + The logs from Squid contain an extra status and hierarchy code appended. DOC_END NAME: access_log cache_access_log @@ -3226,16 +3234,9 @@ DOC_START DOC_END NAME: emulate_httpd_log -COMMENT: on|off -TYPE: onoff -DEFAULT: off -LOC: Config.onoff.common_log +TYPE: obsolete DOC_START - The Cache can emulate the log file format which many 'httpd' - programs use. To disable/enable this emulation, set - emulate_httpd_log to 'off' or 'on'. The default - is to use the native log format since it includes useful - information Squid-specific log analyzers use. + Replace this with an access_log directive using the format 'common' or 'combined'. DOC_END NAME: log_ip_on_direct @@ -3273,27 +3274,15 @@ DOC_START DOC_END NAME: useragent_log -TYPE: string -LOC: Config.Log.useragent -DEFAULT: none -IFDEF: USE_USERAGENT_LOG +TYPE: obsolete DOC_START - Squid will write the User-Agent field from HTTP requests - to the filename specified here. By default useragent_log - is disabled. + Replace this with an access_log directive using the format 'useragent'. DOC_END NAME: referer_log referrer_log -TYPE: string -LOC: Config.Log.referer -DEFAULT: none -IFDEF: USE_REFERER_LOG +TYPE: obsolete DOC_START - Squid will write the Referer field from HTTP requests to the - filename specified here. By default referer_log is disabled. - Note that "referer" is actually a misspelling of "referrer" - however the misspelt version has been accepted into the HTTP RFCs - and we accept both. + Replace this with an access_log directive using the format 'referrer'. DOC_END NAME: pid_filename @@ -3322,14 +3311,9 @@ DOC_START DOC_END NAME: forward_log -IFDEF: WIP_FWD_LOG -TYPE: string -DEFAULT: none -LOC: Config.Log.forward +TYPE: obsolete DOC_START - Logs the server-side requests. - - This is currently work in progress. + Use a regular access.log with ACL limiting it to MISS events. DOC_END NAME: strip_query_terms diff --git a/src/cf_gen_defines b/src/cf_gen_defines index a26081be0d..aa5464ab39 100644 --- a/src/cf_gen_defines +++ b/src/cf_gen_defines @@ -23,11 +23,9 @@ BEGIN { define["USE_ICMP"]="--enable-icmp" define["USE_IDENT"]="--enable-ident-lookups" define["USE_LOADABLE_MODULES"]="--enable-loadable-modules" - define["USE_REFERER_LOG"]="--enable-referer-log" define["USE_SQUID_ESI"]="--enable-esi" define["USE_SSL"]="--enable-ssl" define["USE_UNLINKD"]="--enable-unlinkd" - define["USE_USERAGENT_LOG"]="--enable-useragent-log" define["USE_WCCP"]="--enable-wccp" define["USE_WCCPv2"]="--enable-wccpv2" define["USE_QOS_TOS"]="--enable-zph-qos" diff --git a/src/client_side_request.cc b/src/client_side_request.cc index 76e3d3aadc..0e4f56d6eb 100644 --- a/src/client_side_request.cc +++ b/src/client_side_request.cc @@ -954,21 +954,6 @@ clientInterpretRequestHeaders(ClientHttpRequest * http) s.clean(); } - /** - \todo --enable-useragent-log and --enable-referer-log. We should - probably drop those two as the custom log formats accomplish pretty much the same thing.. - */ -#if USE_USERAGENT_LOG - if ((str = req_hdr->getStr(HDR_USER_AGENT))) - logUserAgent(fqdnFromAddr(http->getConn()->log_addr), str); - -#endif -#if USE_REFERER_LOG - - if ((str = req_hdr->getStr(HDR_REFERER))) - logReferer(fqdnFromAddr(http->getConn()->log_addr), str, http->log_uri); - -#endif #if USE_FORW_VIA_DB if (req_hdr->has(HDR_X_FORWARDED_FOR)) { diff --git a/src/enums.h b/src/enums.h index f5ee494b51..da1e39f68a 100644 --- a/src/enums.h +++ b/src/enums.h @@ -342,20 +342,7 @@ enum { _WIN_OS_WINLON, _WIN_OS_WIN7 }; - -#endif - -typedef enum { - CLF_UNKNOWN, - CLF_AUTO, - CLF_CUSTOM, - CLF_SQUID, - CLF_COMMON, -#if ICAP_CLIENT - CLF_ICAP_SQUID, #endif - CLF_NONE -} customlog_type; enum { DISABLE_PMTU_OFF, diff --git a/src/errorpage.cc b/src/errorpage.cc index 65ccb49495..af4edeedc5 100644 --- a/src/errorpage.cc +++ b/src/errorpage.cc @@ -804,7 +804,7 @@ ErrorState::Convert(char token, bool building_deny_info_url) break; case 't': - mb.Printf("%s", mkhttpdlogtime(&squid_curtime)); + mb.Printf("%s", Time::FormatHttpd(squid_curtime)); break; case 'T': diff --git a/src/forward.cc b/src/forward.cc index 906918105a..bfa6d4792f 100644 --- a/src/forward.cc +++ b/src/forward.cc @@ -70,11 +70,6 @@ static void fwdServerFree(FwdServer * fs); #define MAX_FWD_STATS_IDX 9 static int FwdReplyCodes[MAX_FWD_STATS_IDX + 1][HTTP_INVALID_HEADER + 1]; -#if WIP_FWD_LOG -static void fwdLog(FwdState * fwdState); -static Logfile *logfile = NULL; -#endif - static PconnPool *fwdPconnPool = new PconnPool("server-side"); CBDATA_CLASS_INIT(FwdState); @@ -139,10 +134,6 @@ FwdState::completed() entry->mem_obj->checkUrlChecksum(); #endif -#if WIP_FWD_LOG - - log(); -#endif if (entry->store_status == STORE_PENDING) { if (entry->isEmpty()) { @@ -1266,18 +1257,6 @@ void FwdState::initModule() { memDataInit(MEM_FWD_SERVER, "FwdServer", sizeof(FwdServer), 0); - -#if WIP_FWD_LOG - - if (logfile) - (void) 0; - else if (NULL == Config.Log.forward) - (void) 0; - else - logfile = logfileOpen(Config.Log.forward, 0, 1); - -#endif - RegisterWithCacheManager(); } @@ -1472,47 +1451,3 @@ GetNfmarkToServer(HttpRequest * request) return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch); } - - -/**** WIP_FWD_LOG *************************************************************/ - -#if WIP_FWD_LOG -void -fwdUninit(void) -{ - if (NULL == logfile) - return; - - logfileClose(logfile); - - logfile = NULL; -} - -void -fwdLogRotate(void) -{ - if (logfile) - logfileRotate(logfile); -} - -static void -FwdState::log() -{ - if (NULL == logfile) - return; - - logfilePrintf(logfile, "%9d.%03d %03d %s %s\n", - (int) current_time.tv_sec, - (int) current_time.tv_usec / 1000, - last_status, - RequestMethodStr(request->method), - request->canonical); -} - -void -FwdState::status(http_status s) -{ - last_status = s; -} - -#endif diff --git a/src/forward.h b/src/forward.h index 5cf645bd41..0324c8f4d2 100644 --- a/src/forward.h +++ b/src/forward.h @@ -81,13 +81,6 @@ private: ErrorState *makeConnectingError(const err_type type) const; static void RegisterWithCacheManager(void); -#if WIP_FWD_LOG - - void uninit /**DOCS_NOSEMI*/ - static void logRotate /**DOCS_NOSEMI*/ - void status() /**DOCS_NOSEMI*/ -#endif - public: StoreEntry *entry; HttpRequest *request; @@ -102,10 +95,6 @@ private: time_t start_t; int n_tries; int origin_tries; -#if WIP_FWD_LOG - - http_status last_status; -#endif struct { unsigned int dont_retry:1; diff --git a/src/helper.cc b/src/helper.cc index e0a6998460..6c313cbd1e 100644 --- a/src/helper.cc +++ b/src/helper.cc @@ -35,6 +35,7 @@ #include "squid.h" #include "comm/Write.h" #include "helper.h" +#include "log/Gadgets.h" #include "SquidMath.h" #include "SquidTime.h" #include "Store.h" @@ -441,7 +442,7 @@ helperStats(StoreEntry * sentry, helper * hlp, const char *label) srv->flags.shutdown ? 'S' : ' ', tt < 0.0 ? 0.0 : tt, (int) srv->roffset, - srv->requests[0] ? log_quote(srv->requests[0]->buf) : "(none)"); + srv->requests[0] ? Log::QuoteMimeBlob(srv->requests[0]->buf) : "(none)"); } storeAppendPrintf(sentry, "\nFlags key:\n\n"); @@ -495,7 +496,7 @@ helperStatefulStats(StoreEntry * sentry, statefulhelper * hlp, const char *label srv->request ? (srv->request->placeholder ? 'P' : ' ') : ' ', tt < 0.0 ? 0.0 : tt, (int) srv->roffset, - srv->request ? log_quote(srv->request->buf) : "(none)"); + srv->request ? Log::QuoteMimeBlob(srv->request->buf) : "(none)"); } storeAppendPrintf(sentry, "\nFlags key:\n\n"); diff --git a/src/http.cc b/src/http.cc index 7bb69d10e7..d59403e4dd 100644 --- a/src/http.cc +++ b/src/http.cc @@ -906,10 +906,6 @@ HttpStateData::haveParsedReplyHeaders() entry->mem_obj->vary_headers = xstrdup(vary); } -#if WIP_FWD_LOG - fwdStatus(fwd, s); - -#endif /* * If its not a reply that we will re-forward, then * allow the client to get it. diff --git a/src/log/Config.cc b/src/log/Config.cc index 2360212d49..fd9a148eaf 100644 --- a/src/log/Config.cc +++ b/src/log/Config.cc @@ -1,4 +1,33 @@ #include "config.h" #include "log/Config.h" +#include "log/Tokens.h" +#include "protos.h" Log::LogConfig Log::TheConfig; + +void +Log::LogConfig::parseFormats() +{ + char *name, *def; + + if ((name = strtok(NULL, w_space)) == NULL) + self_destruct(); + + if ((def = strtok(NULL, "\r\n")) == NULL) { + self_destruct(); + return; + } + + debugs(3, 2, "Logformat for '" << name << "' is '" << def << "'"); + + logformat *nlf = new logformat(name); + + if (!accessLogParseLogFormat(&nlf->format, def)) { + self_destruct(); + return; + } + + // add to global config list + nlf->next = logformats; + logformats = nlf; +} diff --git a/src/log/Config.h b/src/log/Config.h index 4a74a26568..f46d4654d0 100644 --- a/src/log/Config.h +++ b/src/log/Config.h @@ -1,17 +1,35 @@ #ifndef SQUID_SRC_LOG_CONFIG_H #define SQUID_SRC_LOG_CONFIG_H +#include "log/Tokens.h" + +class StoreEntry; + namespace Log { class LogConfig { public: + void parseFormats(); + void dumpFormats(StoreEntry *e, const char *name) { + accessLogDumpLogFormat(e, name, logformats); + } + + /// File path to logging daemon executable char *logfile_daemon; + + /// Linked list of custom log formats + logformat *logformats; }; extern LogConfig TheConfig; } // namespace Log +// Legacy parsing wrappers +#define parse_logformat(X) (X)->parseFormats() +#define free_logformat(X) do{ delete (*X).logformats; (*X).logformats=NULL; }while(false) +#define dump_logformat(E,N,D) (D).dumpFormats((E),(N)) + #endif diff --git a/src/log/FormatHttpdCombined.cc b/src/log/FormatHttpdCombined.cc new file mode 100644 index 0000000000..899c92194e --- /dev/null +++ b/src/log/FormatHttpdCombined.cc @@ -0,0 +1,87 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log - Apache combined format + * AUTHOR: Amos Jeffries + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" +#include "AccessLogEntry.h" +#include "HttpRequest.h" +#include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "SquidTime.h" + +void +Log::Format::HttpdCombined(AccessLogEntry * al, Logfile * logfile) +{ + char clientip[MAX_IPSTRLEN]; + + const char *user_ident = FormatName(al->cache.rfc931); + + const char *user_auth = FormatName(al->cache.authuser); + + const char *referer = al->request->header.getStr(HDR_REFERER); + if (!referer || *referer == '\0') + referer = "-"; + + const char *agent = al->request->header.getStr(HDR_USER_AGENT); + if (!agent || *agent == '\0') + agent = "-"; + + logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" \"%s\" \"%s\" %s%s:%s%s", + al->cache.caddr.NtoA(clientip,MAX_IPSTRLEN), + user_ident ? user_ident : dash_str, + user_auth ? user_auth : dash_str, + Time::FormatHttpd(squid_curtime), + al->_private.method_str, + al->url, + al->http.version.major, al->http.version.minor, + al->http.code, + al->cache.replySize, + referer, + agent, + log_tags[al->cache.code], + al->http.statusSfx(), + hier_code_str[al->hier.code], + (Config.onoff.log_mime_hdrs?"":"\n")); + + safe_free(user_ident); + safe_free(user_auth); + + if (Config.onoff.log_mime_hdrs) { + char *ereq = QuoteMimeBlob(al->headers.request); + char *erep = QuoteMimeBlob(al->headers.reply); + logfilePrintf(logfile, " [%s] [%s]\n", ereq, erep); + safe_free(ereq); + safe_free(erep); + } +} diff --git a/src/log/FormatHttpdCommon.cc b/src/log/FormatHttpdCommon.cc new file mode 100644 index 0000000000..3e721b5d1b --- /dev/null +++ b/src/log/FormatHttpdCommon.cc @@ -0,0 +1,74 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log - Apache common format + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" +#include "AccessLogEntry.h" +#include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "SquidTime.h" + +void +Log::Format::HttpdCommon(AccessLogEntry * al, Logfile * logfile) +{ + char clientip[MAX_IPSTRLEN]; + const char *user_auth = FormatName(al->cache.authuser); + const char *user_ident = FormatName(al->cache.rfc931); + + logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" %s%s:%s%s", + al->cache.caddr.NtoA(clientip,MAX_IPSTRLEN), + user_ident ? user_ident : dash_str, + user_auth ? user_auth : dash_str, + Time::FormatHttpd(squid_curtime), + al->_private.method_str, + al->url, + al->http.version.major, al->http.version.minor, + al->http.code, + al->cache.replySize, + log_tags[al->cache.code], + al->http.statusSfx(), + hier_code_str[al->hier.code], + (Config.onoff.log_mime_hdrs?"":"\n")); + + safe_free(user_auth); + safe_free(user_ident); + + if (Config.onoff.log_mime_hdrs) { + char *ereq = QuoteMimeBlob(al->headers.request); + char *erep = QuoteMimeBlob(al->headers.reply); + logfilePrintf(logfile, " [%s] [%s]\n", ereq, erep); + safe_free(ereq); + safe_free(erep); + } +} diff --git a/src/log/FormatSquidCustom.cc b/src/log/FormatSquidCustom.cc new file mode 100644 index 0000000000..a6bed889f7 --- /dev/null +++ b/src/log/FormatSquidCustom.cc @@ -0,0 +1,817 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log - Squid Custom format + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" +#include "AccessLogEntry.h" +#include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "log/Tokens.h" +#include "SquidTime.h" + +#include "MemBuf.h" +#include "HttpRequest.h" +#include "rfc1738.h" +#include "err_detail_type.h" +#include "errorpage.h" + +static void +log_quoted_string(const char *str, char *out) +{ + char *p = out; + + while (*str) { + int l = strcspn(str, "\"\\\r\n\t"); + memcpy(p, str, l); + str += l; + p += l; + + switch (*str) { + + case '\0': + break; + + case '\r': + *p++ = '\\'; + *p++ = 'r'; + str++; + break; + + case '\n': + *p++ = '\\'; + *p++ = 'n'; + str++; + break; + + case '\t': + *p++ = '\\'; + *p++ = 't'; + str++; + break; + + default: + *p++ = '\\'; + *p++ = *str; + str++; + break; + } + } + + *p++ = '\0'; +} + +void +Log::Format::SquidCustom(AccessLogEntry * al, customlog * log) +{ + logformat *lf; + Logfile *logfile; + logformat_token *fmt; + static MemBuf mb; + char tmp[1024]; + String sb; + + mb.reset(); + + lf = log->logFormat; + logfile = log->logfile; + + for (fmt = lf->format; fmt != NULL; fmt = fmt->next) { /* for each token */ + const char *out = NULL; + int quote = 0; + long int outint = 0; + int doint = 0; + int dofree = 0; + int64_t outoff = 0; + int dooff = 0; + + switch (fmt->type) { + + case LFT_NONE: + out = ""; + break; + + case LFT_STRING: + out = fmt->data.string; + break; + + case LFT_CLIENT_IP_ADDRESS: + if (al->cache.caddr.IsNoAddr()) // e.g., ICAP OPTIONS lack client + out = "-"; + else + out = al->cache.caddr.NtoA(tmp,1024); + break; + + case LFT_CLIENT_FQDN: + if (al->cache.caddr.IsAnyAddr()) // e.g., ICAP OPTIONS lack client + out = "-"; + else + out = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); + if (!out) { + out = al->cache.caddr.NtoA(tmp,1024); + } + + break; + + case LFT_CLIENT_PORT: + if (al->request) { + outint = al->request->client_addr.GetPort(); + doint = 1; + } + break; + +#if USE_SQUID_EUI + case LFT_CLIENT_EUI: + if (al->request) { + if (al->cache.caddr.IsIPv4()) + al->request->client_eui48.encode(tmp, 1024); + else + al->request->client_eui64.encode(tmp, 1024); + out = tmp; + } + break; +#endif + + /* case LFT_SERVER_IP_ADDRESS: */ + + case LFT_SERVER_IP_OR_PEER_NAME: + out = al->hier.host; + + break; + + /* case LFT_SERVER_PORT: */ + + case LFT_LOCAL_IP: + if (al->request) { + out = al->request->my_addr.NtoA(tmp,1024); + } + + break; + + case LFT_LOCAL_PORT: + if (al->request) { + outint = al->request->my_addr.GetPort(); + doint = 1; + } + + break; + + case LFT_PEER_LOCAL_PORT: + if (al->hier.peer_local_port) { + outint = al->hier.peer_local_port; + doint = 1; + } + + break; + + case LFT_TIME_SECONDS_SINCE_EPOCH: + // some platforms store time in 32-bit, some 64-bit... + outoff = static_cast(current_time.tv_sec); + dooff = 1; + break; + + case LFT_TIME_SUBSECOND: + outint = current_time.tv_usec / fmt->divisor; + doint = 1; + break; + + + case LFT_TIME_LOCALTIME: + + case LFT_TIME_GMT: { + const char *spec; + + struct tm *t; + spec = fmt->data.timespec; + + if (fmt->type == LFT_TIME_LOCALTIME) { + if (!spec) + spec = "%d/%b/%Y:%H:%M:%S %z"; + t = localtime(&squid_curtime); + } else { + if (!spec) + spec = "%d/%b/%Y:%H:%M:%S"; + + t = gmtime(&squid_curtime); + } + + strftime(tmp, sizeof(tmp), spec, t); + + out = tmp; + } + + break; + + case LFT_TIME_TO_HANDLE_REQUEST: + outint = al->cache.msec; + doint = 1; + break; + + case LFT_PEER_RESPONSE_TIME: + if (al->hier.peer_response_time < 0) { + out = "-"; + } else { + outoff = al->hier.peer_response_time; + dooff = 1; + } + break; + + case LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME: + if (al->hier.total_response_time < 0) { + out = "-"; + } else { + outoff = al->hier.total_response_time; + dooff = 1; + } + break; + + case LFT_DNS_WAIT_TIME: + if (al->request && al->request->dnsWait >= 0) { + outint = al->request->dnsWait; + doint = 1; + } + break; + + case LFT_REQUEST_HEADER: + + if (al->request) + sb = al->request->header.getByName(fmt->data.header.header); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ADAPTED_REQUEST_HEADER: + + if (al->request) + sb = al->adapted_request->header.getByName(fmt->data.header.header); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_REPLY_HEADER: + if (al->reply) + sb = al->reply->header.getByName(fmt->data.header.header); + + out = sb.termedBuf(); + + quote = 1; + + break; + +#if USE_ADAPTATION + case LTF_ADAPTATION_SUM_XACT_TIMES: + if (al->request) { + Adaptation::History::Pointer ah = al->request->adaptHistory(); + if (ah != NULL) + ah->sumLogString(fmt->data.string, sb); + out = sb.termedBuf(); + } + break; + + case LTF_ADAPTATION_ALL_XACT_TIMES: + if (al->request) { + Adaptation::History::Pointer ah = al->request->adaptHistory(); + if (ah != NULL) + ah->allLogString(fmt->data.string, sb); + out = sb.termedBuf(); + } + break; +#endif + +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + if (al->request) { + Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); + if (ih != NULL) + sb = ih->mergeOfIcapHeaders.getByName(fmt->data.header.header); + } + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: + if (al->request) { + Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); + if (ih != NULL) + sb = ih->mergeOfIcapHeaders.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + } + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: + out = al->headers.icap; + + quote = 1; + + break; + + case LFT_ICAP_ADDR: + if (!out) + out = al->icap.hostAddr.NtoA(tmp,1024); + break; + + case LFT_ICAP_SERV_NAME: + out = al->icap.serviceName.termedBuf(); + break; + + case LFT_ICAP_REQUEST_URI: + out = al->icap.reqUri.termedBuf(); + break; + + case LFT_ICAP_REQUEST_METHOD: + out = Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod); + break; + + case LFT_ICAP_BYTES_SENT: + outoff = al->icap.bytesSent; + dooff = 1; + break; + + case LFT_ICAP_BYTES_READ: + outoff = al->icap.bytesRead; + dooff = 1; + break; + + case LFT_ICAP_BODY_BYTES_READ: + if (al->icap.bodyBytesRead >= 0) { + outoff = al->icap.bodyBytesRead; + dooff = 1; + } + // else if icap.bodyBytesRead < 0, we do not have any http data, + // so just print a "-" (204 responses etc) + break; + + case LFT_ICAP_REQ_HEADER: + if (NULL != al->icap.request) { + sb = al->icap.request->header.getByName(fmt->data.header.header); + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REQ_HEADER_ELEM: + if (al->request) + sb = al->icap.request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_REQ_ALL_HEADERS: + if (al->icap.request) { + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = al->icap.request->header.getEntry(&pos)) { + sb.append(e->name); + sb.append(": "); + sb.append(e->value); + sb.append("\r\n"); + } + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REP_HEADER: + if (NULL != al->icap.reply) { + sb = al->icap.reply->header.getByName(fmt->data.header.header); + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_REP_HEADER_ELEM: + if (NULL != al->icap.reply) + sb = al->icap.reply->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ICAP_REP_ALL_HEADERS: + if (al->icap.reply) { + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = al->icap.reply->header.getEntry(&pos)) { + sb.append(e->name); + sb.append(": "); + sb.append(e->value); + sb.append("\r\n"); + } + out = sb.termedBuf(); + quote = 1; + } + break; + + case LFT_ICAP_TR_RESPONSE_TIME: + outint = al->icap.trTime; + doint = 1; + break; + + case LFT_ICAP_IO_TIME: + outint = al->icap.ioTime; + doint = 1; + break; + + case LFT_ICAP_STATUS_CODE: + outint = al->icap.resStatus; + doint = 1; + break; + + case LFT_ICAP_OUTCOME: + out = al->icap.outcome; + break; + + case LFT_ICAP_TOTAL_TIME: + outint = al->icap.processingTime; + doint = 1; + break; +#endif + case LFT_REQUEST_HEADER_ELEM: + if (al->request) + sb = al->request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_ADAPTED_REQUEST_HEADER_ELEM: + if (al->adapted_request) + sb = al->adapted_request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_REPLY_HEADER_ELEM: + if (al->reply) + sb = al->reply->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); + + out = sb.termedBuf(); + + quote = 1; + + break; + + case LFT_REQUEST_ALL_HEADERS: + out = al->headers.request; + + quote = 1; + + break; + + case LFT_ADAPTED_REQUEST_ALL_HEADERS: + out = al->headers.adapted_request; + + quote = 1; + + break; + + case LFT_REPLY_ALL_HEADERS: + out = al->headers.reply; + + quote = 1; + + break; + + case LFT_USER_NAME: + out = Log::FormatName(al->cache.authuser); + + if (!out) + out = Log::FormatName(al->cache.extuser); + +#if USE_SSL + + if (!out) + out = Log::FormatName(al->cache.ssluser); + +#endif + + if (!out) + out = Log::FormatName(al->cache.rfc931); + + dofree = 1; + + break; + + case LFT_USER_LOGIN: + out = Log::FormatName(al->cache.authuser); + + dofree = 1; + + break; + + case LFT_USER_IDENT: + out = Log::FormatName(al->cache.rfc931); + + dofree = 1; + + break; + + case LFT_USER_EXTERNAL: + out = Log::FormatName(al->cache.extuser); + + dofree = 1; + + break; + + /* case LFT_USER_REALM: */ + /* case LFT_USER_SCHEME: */ + + // the fmt->type can not be LFT_HTTP_SENT_STATUS_CODE_OLD_30 + // but compiler complains if ommited + case LFT_HTTP_SENT_STATUS_CODE_OLD_30: + case LFT_HTTP_SENT_STATUS_CODE: + outint = al->http.code; + + doint = 1; + + break; + + case LFT_HTTP_RECEIVED_STATUS_CODE: + if (al->hier.peer_reply_status == HTTP_STATUS_NONE) { + out = "-"; + } else { + outint = al->hier.peer_reply_status; + doint = 1; + } + break; + /* case LFT_HTTP_STATUS: + * out = statusline->text; + * quote = 1; + * break; + */ + case LFT_HTTP_BODY_BYTES_READ: + if (al->hier.bodyBytesRead >= 0) { + outoff = al->hier.bodyBytesRead; + dooff = 1; + } + // else if hier.bodyBytesRead < 0 we did not have any data exchange with + // a peer server so just print a "-" (eg requests served from cache, + // or internal error messages). + break; + + case LFT_SQUID_STATUS: + if (al->http.timedout || al->http.aborted) { + snprintf(tmp, sizeof(tmp), "%s%s", log_tags[al->cache.code], + al->http.statusSfx()); + out = tmp; + } else { + out = log_tags[al->cache.code]; + } + + break; + + case LFT_SQUID_ERROR: + if (al->request && al->request->errType != ERR_NONE) + out = errorPageName(al->request->errType); + break; + + case LFT_SQUID_ERROR_DETAIL: + if (al->request && al->request->errDetail != ERR_DETAIL_NONE) { + if (al->request->errDetail > ERR_DETAIL_START && + al->request->errDetail < ERR_DETAIL_MAX) + out = errorDetailName(al->request->errDetail); + else { + if (al->request->errDetail >= ERR_DETAIL_EXCEPTION_START) + snprintf(tmp, sizeof(tmp), "%s=0x%X", + errorDetailName(al->request->errDetail), (uint32_t) al->request->errDetail); + else + snprintf(tmp, sizeof(tmp), "%s=%d", + errorDetailName(al->request->errDetail), al->request->errDetail); + out = tmp; + } + } + break; + + case LFT_SQUID_HIERARCHY: + if (al->hier.ping.timedout) + mb.append("TIMEOUT_", 8); + + out = hier_code_str[al->hier.code]; + + break; + + case LFT_MIME_TYPE: + out = al->http.content_type; + + break; + + case LFT_REQUEST_METHOD: + out = al->_private.method_str; + + break; + + case LFT_REQUEST_URI: + out = al->url; + + break; + + case LFT_REQUEST_URLPATH: + if (al->request) { + out = al->request->urlpath.termedBuf(); + quote = 1; + } + break; + + case LFT_REQUEST_VERSION: + snprintf(tmp, sizeof(tmp), "%d.%d", (int) al->http.version.major, (int) al->http.version.minor); + out = tmp; + break; + + case LFT_REQUEST_SIZE_TOTAL: + outoff = al->cache.requestSize; + dooff = 1; + break; + + /*case LFT_REQUEST_SIZE_LINE: */ + case LFT_REQUEST_SIZE_HEADERS: + outoff = al->cache.requestHeadersSize; + dooff =1; + break; + /*case LFT_REQUEST_SIZE_BODY: */ + /*case LFT_REQUEST_SIZE_BODY_NO_TE: */ + + case LFT_REPLY_SIZE_TOTAL: + outoff = al->cache.replySize; + dooff = 1; + break; + + case LFT_REPLY_HIGHOFFSET: + outoff = al->cache.highOffset; + + dooff = 1; + + break; + + case LFT_REPLY_OBJECTSIZE: + outoff = al->cache.objectSize; + + dooff = 1; + + break; + + /*case LFT_REPLY_SIZE_LINE: */ + case LFT_REPLY_SIZE_HEADERS: + outint = al->cache.replyHeadersSize; + doint = 1; + break; + /*case LFT_REPLY_SIZE_BODY: */ + /*case LFT_REPLY_SIZE_BODY_NO_TE: */ + + case LFT_TAG: + if (al->request) + out = al->request->tag.termedBuf(); + + quote = 1; + + break; + + case LFT_IO_SIZE_TOTAL: + outint = al->cache.requestSize + al->cache.replySize; + doint = 1; + break; + + case LFT_EXT_LOG: + if (al->request) + out = al->request->extacl_log.termedBuf(); + + quote = 1; + + break; + + case LFT_SEQUENCE_NUMBER: + outoff = logfile->sequence_number; + dooff = 1; + break; + + case LFT_PERCENT: + out = "%"; + + break; + } + + if (dooff) { + snprintf(tmp, sizeof(tmp), "%0*" PRId64, fmt->zero ? (int) fmt->width : 0, outoff); + out = tmp; + + } else if (doint) { + snprintf(tmp, sizeof(tmp), "%0*ld", fmt->zero ? (int) fmt->width : 0, outint); + out = tmp; + } + + if (out && *out) { + if (quote || fmt->quote != LOG_QUOTE_NONE) { + char *newout = NULL; + int newfree = 0; + + switch (fmt->quote) { + + case LOG_QUOTE_NONE: + newout = rfc1738_escape_unescaped(out); + break; + + case LOG_QUOTE_QUOTES: + size_t out_len = static_cast(strlen(out)) * 2 + 1; + if (out_len >= sizeof(tmp)) { + newout = (char *)xmalloc(out_len); + newfree = 1; + } else + newout = tmp; + log_quoted_string(out, newout); + break; + + case LOG_QUOTE_MIMEBLOB: + newout = Log::QuoteMimeBlob(out); + newfree = 1; + break; + + case LOG_QUOTE_URL: + newout = rfc1738_escape(out); + break; + + case LOG_QUOTE_RAW: + break; + } + + if (newout) { + if (dofree) + safe_free(out); + + out = newout; + + dofree = newfree; + } + } + + if (fmt->width) { + if (fmt->left) + mb.Printf("%-*s", (int) fmt->width, out); + else + mb.Printf("%*s", (int) fmt->width, out); + } else + mb.append(out, strlen(out)); + } else { + mb.append("-", 1); + } + + if (fmt->space) + mb.append(" ", 1); + + sb.clean(); + + if (dofree) + safe_free(out); + } + + logfilePrintf(logfile, "%s\n", mb.buf); +} diff --git a/src/log/FormatSquidIcap.cc b/src/log/FormatSquidIcap.cc new file mode 100644 index 0000000000..5fccc88f9e --- /dev/null +++ b/src/log/FormatSquidIcap.cc @@ -0,0 +1,91 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log - Squid ICAP Logging + * AUTHOR: Alex Rousskov + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" + +#if ICAP_CLIENT + +#include "AccessLogEntry.h" +#include "HttpRequest.h" +#include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "SquidTime.h" + +void +Log::Format::SquidIcap(AccessLogEntry * al, Logfile * logfile) +{ + const char *client = NULL; + const char *user = NULL; + char tmp[MAX_IPSTRLEN], clientbuf[MAX_IPSTRLEN]; + + if (al->cache.caddr.IsAnyAddr()) { // ICAP OPTIONS xactions lack client + client = "-"; + } else { + if (Config.onoff.log_fqdn) + client = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); + if (!client) + client = al->cache.caddr.NtoA(clientbuf, MAX_IPSTRLEN); + } + + user = Log::FormatName(al->cache.authuser); + + if (!user) + user = Log::FormatName(al->cache.extuser); + +#if USE_SSL + if (!user) + user = Log::FormatName(al->cache.ssluser); +#endif + + if (!user) + user = Log::FormatName(al->cache.rfc931); + + if (user && !*user) + safe_free(user); + + logfilePrintf(logfile, "%9ld.%03d %6d %s -/%03d %"PRId64" %s %s %s -/%s -\n", + (long int) current_time.tv_sec, + (int) current_time.tv_usec / 1000, + al->icap.trTime, + client, + al->icap.resStatus, + al->icap.bytesRead, + Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod), + al->icap.reqUri.termedBuf(), + user ? user : "-", + al->icap.hostAddr.NtoA(tmp, MAX_IPSTRLEN)); + safe_free(user); +} +#endif diff --git a/src/log/FormatSquidNative.cc b/src/log/FormatSquidNative.cc new file mode 100644 index 0000000000..297b6ccdbc --- /dev/null +++ b/src/log/FormatSquidNative.cc @@ -0,0 +1,91 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log - Squid format + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" +#include "AccessLogEntry.h" +#include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "SquidTime.h" + +void +Log::Format::SquidNative(AccessLogEntry * al, Logfile * logfile) +{ + const char *user = NULL; + char clientip[MAX_IPSTRLEN]; + + user = FormatName(al->cache.authuser); + + if (!user) + user = FormatName(al->cache.extuser); + +#if USE_SSL + if (!user) + user = FormatName(al->cache.ssluser); +#endif + + if (!user) + user = FormatName(al->cache.rfc931); + + if (user && !*user) + safe_free(user); + + logfilePrintf(logfile, "%9ld.%03d %6d %s %s%s/%03d %"PRId64" %s %s %s %s%s/%s %s%s", + (long int) current_time.tv_sec, + (int) current_time.tv_usec / 1000, + al->cache.msec, + al->cache.caddr.NtoA(clientip, MAX_IPSTRLEN), + log_tags[al->cache.code], + al->http.statusSfx(), + al->http.code, + al->cache.replySize, + al->_private.method_str, + al->url, + user ? user : dash_str, + al->hier.ping.timedout ? "TIMEOUT_" : "", + hier_code_str[al->hier.code], + al->hier.host, + al->http.content_type, + (Config.onoff.log_mime_hdrs?"":"\n")); + + safe_free(user); + + if (Config.onoff.log_mime_hdrs) { + char *ereq = QuoteMimeBlob(al->headers.request); + char *erep = QuoteMimeBlob(al->headers.reply); + logfilePrintf(logfile, " [%s] [%s]\n", ereq, erep); + safe_free(ereq); + safe_free(erep); + } +} diff --git a/src/referer.cc b/src/log/FormatSquidReferer.cc similarity index 59% rename from src/referer.cc rename to src/log/FormatSquidReferer.cc index 7a1eb5d398..585187dd65 100644 --- a/src/referer.cc +++ b/src/log/FormatSquidReferer.cc @@ -1,9 +1,9 @@ /* * $Id$ * - * DEBUG: section 40 Referer Logging + * DEBUG: section 46 Access Log - Squid referer format * AUTHOR: Joe Ramey (useragent) - * Jens-S. Vöckler (mod 4 referer) + * Jens-S. V?ckler (mod 4 referer) * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -33,71 +33,28 @@ * */ -#include "squid.h" +#include "config.h" +#include "AccessLogEntry.h" +#include "HttpRequest.h" #include "log/File.h" +#include "log/Formats.h" #include "SquidTime.h" -#if USE_REFERER_LOG -static Logfile *refererlog = NULL; -#endif - void -refererOpenLog(void) +Log::Format::SquidReferer(AccessLogEntry *al, Logfile *logfile) { -#if USE_REFERER_LOG - assert(NULL == refererlog); + const char *referer = al->request->header.getStr(HDR_REFERER); - if (!Config.Log.referer || (0 == strcmp(Config.Log.referer, "none"))) { - debugs(40, 1, "Referer logging is disabled."); + // do not log unless there is something to be displayed + if (!referer || *referer == '\0') return; - } - - refererlog = logfileOpen(Config.Log.referer, 0, 1); -#endif -} -void -refererRotateLog(void) -{ -#if USE_REFERER_LOG - - if (NULL == refererlog) - return; - - logfileRotate(refererlog); - -#endif -} - -void -logReferer(const char *client, const char *referer, const char *uri) -{ -#if USE_REFERER_LOG + char clientip[MAX_IPSTRLEN]; - if (NULL == refererlog) - return; - - logfilePrintf(refererlog, "%9d.%03d %s %s %s\n", - (int) current_time.tv_sec, + logfilePrintf(logfile, "%9ld.%03d %s %s %s\n", + (long int) current_time.tv_sec, (int) current_time.tv_usec / 1000, - client, + al->cache.caddr.NtoA(clientip, MAX_IPSTRLEN), referer, - uri ? uri : "-"); - -#endif -} - -void -refererCloseLog(void) -{ -#if USE_REFERER_LOG - - if (NULL == refererlog) - return; - - logfileClose(refererlog); - - refererlog = NULL; - -#endif + al->url ? al->url : "-"); } diff --git a/src/useragent.cc b/src/log/FormatSquidUseragent.cc similarity index 55% rename from src/useragent.cc rename to src/log/FormatSquidUseragent.cc index 996b7f8700..ce9f676a1a 100644 --- a/src/useragent.cc +++ b/src/log/FormatSquidUseragent.cc @@ -1,8 +1,9 @@ /* * $Id$ * - * DEBUG: section 40 User-Agent Logging + * DEBUG: section 46 Access Log - Squid useragent format * AUTHOR: Joe Ramey + * AUTHOR: Amos Jeffries * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -32,77 +33,26 @@ * */ -#include "squid.h" +#include "config.h" +#include "AccessLogEntry.h" +#include "HttpRequest.h" #include "log/File.h" +#include "log/Formats.h" #include "SquidTime.h" -#if USE_USERAGENT_LOG -static Logfile *useragentlog = NULL; -#endif - void -useragentOpenLog(void) +Log::Format::SquidUserAgent(AccessLogEntry * al, Logfile * logfile) { -#if USE_USERAGENT_LOG - assert(NULL == useragentlog); - - if (!Config.Log.useragent || (0 == strcmp(Config.Log.useragent, "none"))) { - debugs(40, 1, "User-Agent logging is disabled."); - return; - } + char clientip[MAX_IPSTRLEN]; - useragentlog = logfileOpen(Config.Log.useragent, 0, 1); -#endif -} - -void -useragentRotateLog(void) -{ -#if USE_USERAGENT_LOG + const char *agent = al->request->header.getStr(HDR_USER_AGENT); - if (NULL == useragentlog) + // do not log unless there is something to be displayed. + if (!agent || *agent == '\0') return; - logfileRotate(useragentlog); - -#endif -} - -void -logUserAgent(const char *client, const char *agent) -{ -#if USE_USERAGENT_LOG - static time_t last_time = 0; - static char time_str[128]; - const char *s; - - if (NULL == useragentlog) - return; - - if (squid_curtime != last_time) { - s = mkhttpdlogtime(&squid_curtime); - strcpy(time_str, s); - last_time = squid_curtime; - } - - logfilePrintf(useragentlog, "%s [%s] \"%s\"\n", - client, - time_str, + logfilePrintf(logfile, "%s [%s] \"%s\"\n", + al->cache.caddr.NtoA(clientip,MAX_IPSTRLEN), + Time::FormatHttpd(squid_curtime), agent); -#endif -} - -void -useragentLogClose(void) -{ -#if USE_USERAGENT_LOG - - if (NULL == useragentlog) - return; - - logfileClose(useragentlog); - - useragentlog = NULL; - -#endif } diff --git a/src/log/Formats.h b/src/log/Formats.h new file mode 100644 index 0000000000..97ac52c48f --- /dev/null +++ b/src/log/Formats.h @@ -0,0 +1,51 @@ +#ifndef _SQUID_LOG_FORMATS_H +#define _SQUID_LOG_FORMATS_H + +class AccessLogEntry; +class Logfile; + +namespace Log +{ + +namespace Format +{ + +typedef enum { + CLF_UNKNOWN, + CLF_COMBINED, + CLF_COMMON, + CLF_CUSTOM, +#if ICAP_CLIENT + CLF_ICAP_SQUID, +#endif + CLF_REFERER, + CLF_SQUID, + CLF_USERAGENT, + CLF_NONE +} log_type; + +/// Native Squid Format Display +void SquidNative(AccessLogEntry * al, Logfile * logfile); + +/// Display log details in Squid ICAP format. +void SquidIcap(AccessLogEntry * al, Logfile * logfile); + +/// Display log details in useragent format. +void SquidUserAgent(AccessLogEntry * al, Logfile * logfile); + +/// Display log details in Squid old refererlog format. +void SquidReferer(AccessLogEntry * al, Logfile * logfile); + +/// Log with a local custom format +void SquidCustom(AccessLogEntry * al, customlog * log); + +/// Log with Apache httpd common format +void HttpdCommon(AccessLogEntry * al, Logfile * logfile); + +/// Log with Apache httpd combined format +void HttpdCombined(AccessLogEntry * al, Logfile * logfile); + +}; // namespace Format +}; // namespace Log + +#endif /* _SQUID_LOG_FORMATS_H */ diff --git a/src/log/Gadgets.cc b/src/log/Gadgets.cc new file mode 100644 index 0000000000..f188f813e7 --- /dev/null +++ b/src/log/Gadgets.cc @@ -0,0 +1,156 @@ +#include "config.h" +#include "log/Gadgets.h" + +static const char c2x[] = + "000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + "505152535455565758595a5b5c5d5e5f" + "606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + +#if DEAD_USING_QUOTEMIMEBLOB +/** copy of Log::QuoteMimeBlob. Bugs there will be found here. + * This omits [] characters but is otherwise identical to Log::QuoteMimeBlob when OLD_LOG_MIME = 1 + */ +static char * +username_quote(const char *header) +{ + int c; + int i; + char *buf; + char *buf_cursor; + + if (header == NULL) { + buf = static_cast(xcalloc(1, 1)); + *buf = '\0'; + return buf; + } + + buf = static_cast(xcalloc(1, (strlen(header) * 3) + 1)); + buf_cursor = buf; + /* + * We escape: space \x00-\x1F and space (0x40) and \x7F-\xFF + * to prevent garbage in the logs. CR and LF are also there just in case. + */ + + while ((c = *(const unsigned char *) header++) != '\0') { + if (c == '\r') { + *buf_cursor++ = '\\'; + *buf_cursor++ = 'r'; + } else if (c == '\n') { + *buf_cursor++ = '\\'; + *buf_cursor++ = 'n'; + } else if (c <= 0x1F + || c >= 0x7F + || c == '%' + || c == ' ') { + *buf_cursor++ = '%'; + i = c * 2; + *buf_cursor++ = c2x[i]; + *buf_cursor++ = c2x[i + 1]; + } else { + *buf_cursor++ = (char) c; + } + } + + *buf_cursor = '\0'; + return buf; +} +#endif // DEAD + +char * +Log::FormatName(const char *name) +{ + if (NULL == name) + return NULL; + + if (name[0] == '\0') + return NULL; + + return QuoteMimeBlob(name); +// return username_quote(name); +} + +char * +Log::QuoteMimeBlob(const char *header) +{ + int c; + int i; + char *buf; + char *buf_cursor; + + if (header == NULL) { + buf = static_cast(xcalloc(1, 1)); + *buf = '\0'; + return buf; + } + + buf = static_cast(xcalloc(1, (strlen(header) * 3) + 1)); + buf_cursor = buf; + /** + * Whe OLD_LOG_MIME is defined we escape: \x00-\x1F"#%;<>?{}|\\\\^~`\[\]\x7F-\xFF + * which is the default escape list for the CPAN Perl5 URI module + * modulo the inclusion of space (x40) to make the raw logs a bit + * more readable. + */ + + while ((c = *(const unsigned char *) header++) != '\0') { +#if !OLD_LOG_MIME + if (c == '\r') { + *buf_cursor++ = '\\'; + *buf_cursor++ = 'r'; + } else if (c == '\n') { + *buf_cursor++ = '\\'; + *buf_cursor++ = 'n'; + } else +#endif + if (c <= 0x1F + || c >= 0x7F + || c == '%' +#if OLD_LOG_MIME + || c == '"' + || c == '#' + || c == ';' + || c == '<' + || c == '>' + || c == '?' + || c == '{' + || c == '}' + || c == '|' + || c == '\\' + || c == '^' + || c == '~' + || c == '`' +#endif + || c == '[' + || c == ']') { + *buf_cursor++ = '%'; + i = c * 2; + *buf_cursor++ = c2x[i]; + *buf_cursor++ = c2x[i + 1]; +#if !OLD_LOG_MIME + + } else if (c == '\\') { + *buf_cursor++ = '\\'; + *buf_cursor++ = '\\'; +#endif + + } else { + *buf_cursor++ = (char) c; + } + } + + *buf_cursor = '\0'; + return buf; +} diff --git a/src/log/Gadgets.h b/src/log/Gadgets.h new file mode 100644 index 0000000000..9e5e1dd0d9 --- /dev/null +++ b/src/log/Gadgets.h @@ -0,0 +1,19 @@ +#ifndef _SQUID_LOG_GADGETS_H +#define _SQUID_LOG_GADGETS_H + +namespace Log +{ + +/// Safely URL-encode a username. +/// Accepts NULL or empty strings. +char * FormatName(const char *name); + +/** URL-style encoding on a MIME headers blob. + * May accept NULL or empty strings. + * \return A dynamically allocated string. recipient is responsible for free()'ing + */ +char *QuoteMimeBlob(const char *header); + +}; // namespace Log + +#endif /* _SQUID_LOG_GADGETS_H */ diff --git a/src/log/Makefile.am b/src/log/Makefile.am index 82ffb74846..cab6f05580 100644 --- a/src/log/Makefile.am +++ b/src/log/Makefile.am @@ -9,6 +9,16 @@ liblog_la_SOURCES = \ Config.h \ File.cc \ File.h \ + FormatHttpdCombined.cc \ + FormatHttpdCommon.cc \ + Formats.h \ + FormatSquidCustom.cc \ + FormatSquidIcap.cc \ + FormatSquidNative.cc \ + FormatSquidReferer.cc \ + FormatSquidUseragent.cc \ + Gadgets.cc \ + Gadgets.h \ ModDaemon.cc \ ModDaemon.h \ ModStdio.cc \ @@ -18,4 +28,7 @@ liblog_la_SOURCES = \ ModTcp.cc \ ModTcp.h \ ModUdp.cc \ - ModUdp.h + ModUdp.h \ + Tokens.cc \ + Tokens.h + diff --git a/src/log/Tokens.cc b/src/log/Tokens.cc new file mode 100644 index 0000000000..4afb57b31f --- /dev/null +++ b/src/log/Tokens.cc @@ -0,0 +1,690 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log Format Tokens + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "config.h" +#include "log/Tokens.h" +#include "Store.h" + +const char *log_tags[] = { + "NONE", + "TCP_HIT", + "TCP_MISS", + "TCP_REFRESH_UNMODIFIED", + "TCP_REFRESH_FAIL", // same tag logged for LOG_TCP_REFRESH_FAIL_OLD and + "TCP_REFRESH_FAIL", // LOG_TCP_REFRESH_FAIL_ERR for backward-compatibility + "TCP_REFRESH_MODIFIED", + "TCP_CLIENT_REFRESH_MISS", + "TCP_IMS_HIT", + "TCP_SWAPFAIL_MISS", + "TCP_NEGATIVE_HIT", + "TCP_MEM_HIT", + "TCP_DENIED", + "TCP_DENIED_REPLY", + "TCP_OFFLINE_HIT", +#if LOG_TCP_REDIRECTS + "TCP_REDIRECT", +#endif + "UDP_HIT", + "UDP_MISS", + "UDP_DENIED", + "UDP_INVALID", + "UDP_MISS_NOFETCH", + "ICP_QUERY", + "LOG_TYPE_MAX" +}; + +#if USE_ADAPTATION +bool alLogformatHasAdaptToken = false; +#endif + +#if ICAP_CLIENT +bool alLogformatHasIcapToken = false; +#endif + +struct logformat_token_table_entry logformat_token_table[] = { + + {">a", LFT_CLIENT_IP_ADDRESS}, + {">p", LFT_CLIENT_PORT}, + {">A", LFT_CLIENT_FQDN}, +#if USE_SQUID_EUI + {">eui", LFT_CLIENT_EUI}, +#endif + + /*{ "ha", LFT_ADAPTED_REQUEST_HEADER}, + {">ha", LFT_ADAPTED_REQUEST_ALL_HEADERS}, + {">h", LFT_REQUEST_HEADER}, + {">h", LFT_REQUEST_ALL_HEADERS}, + {"Hs", LFT_HTTP_SENT_STATUS_CODE}, + {"v", LFT_REQUEST_VERSION}, + {"rv", LFT_REQUEST_VERSION}, + + { ">st", LFT_REQUEST_SIZE_TOTAL }, + /*{ ">sl", LFT_REQUEST_SIZE_LINE }, * / / * the request line "GET ... " */ + { ">sh", LFT_REQUEST_SIZE_HEADERS }, + /*{ ">sb", LFT_REQUEST_SIZE_BODY }, */ + /*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */ + + {"st", LFT_ICAP_BYTES_SENT}, + {"icap::h", LFT_ICAP_REQ_HEADER}, + {"icap:: 0) { + char *cp; + /* it's a string for sure, until \0 or the next % */ + cp = (char *)xmalloc(l + 1); + xstrncpy(cp, cur, l + 1); + lt->type = LFT_STRING; + lt->data.string = cp; + + while (l > 0) { + switch (*cur) { + + case '"': + + if (*quote == LOG_QUOTE_NONE) + *quote = LOG_QUOTE_QUOTES; + else if (*quote == LOG_QUOTE_QUOTES) + *quote = LOG_QUOTE_NONE; + + break; + + case '[': + if (*quote == LOG_QUOTE_NONE) + *quote = LOG_QUOTE_MIMEBLOB; + + break; + + case ']': + if (*quote == LOG_QUOTE_MIMEBLOB) + *quote = LOG_QUOTE_NONE; + + break; + } + + cur++; + l--; + } + + goto done; + } + + if (!*cur) + goto done; + + cur++; + + switch (*cur) { + + case '"': + lt->quote = LOG_QUOTE_QUOTES; + cur++; + break; + + case '\'': + lt->quote = LOG_QUOTE_RAW; + cur++; + break; + + case '[': + lt->quote = LOG_QUOTE_MIMEBLOB; + cur++; + break; + + case '#': + lt->quote = LOG_QUOTE_URL; + cur++; + break; + + default: + lt->quote = *quote; + break; + } + + if (*cur == '-') { + lt->left = 1; + cur++; + } + + if (*cur == '0') { + lt->zero = 1; + cur++; + } + + if (xisdigit(*cur)) + lt->width = strtol(cur, &cur, 10); + + if (*cur == '.') + lt->precision = strtol(cur + 1, &cur, 10); + + if (*cur == '{') { + char *cp; + cur++; + l = strcspn(cur, "}"); + cp = (char *)xmalloc(l + 1); + xstrncpy(cp, cur, l + 1); + lt->data.string = cp; + cur += l; + + if (*cur == '}') + cur++; + } + + // For upward compatibility, assume "http::" prefix as default prefix + // for all log access formating codes, except those starting + // from "icap::", "adapt::" and "%" + if (strncmp(cur,"http::", 6) == 0 && + strncmp(cur+6, "icap::", 6) != 0 && + strncmp(cur+6, "adapt::", 12) != 0 && *(cur+6) != '%' ) { + cur += 6; + } + + lt->type = LFT_NONE; + + for (lte = logformat_token_table; lte->config != NULL; lte++) { + if (strncmp(lte->config, cur, strlen(lte->config)) == 0) { + lt->type = lte->token_type; + cur += strlen(lte->config); + break; + } + } + + if (lt->type == LFT_NONE) { + fatalf("Can't parse configuration token: '%s'\n", + def); + } + + if (*cur == ' ') { + lt->space = 1; + cur++; + } + +done: + + switch (lt->type) { + +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + + case LFT_ICAP_REQ_HEADER: + + case LFT_ICAP_REP_HEADER: +#endif + + case LFT_ADAPTED_REQUEST_HEADER: + + case LFT_REQUEST_HEADER: + + case LFT_REPLY_HEADER: + + if (lt->data.string) { + char *header = lt->data.string; + char *cp = strchr(header, ':'); + + if (cp) { + *cp++ = '\0'; + + if (*cp == ',' || *cp == ';' || *cp == ':') + lt->data.header.separator = *cp++; + else + lt->data.header.separator = ','; + + lt->data.header.element = cp; + + switch (lt->type) { + case LFT_REQUEST_HEADER: + lt->type = LFT_REQUEST_HEADER_ELEM; + break; + + case LFT_ADAPTED_REQUEST_HEADER: + lt->type = LFT_ADAPTED_REQUEST_HEADER_ELEM; + break; + + case LFT_REPLY_HEADER: + lt->type = LFT_REPLY_HEADER_ELEM; + break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + lt->type = LFT_ICAP_LAST_MATCHED_HEADER_ELEM; + break; + case LFT_ICAP_REQ_HEADER: + lt->type = LFT_ICAP_REQ_HEADER_ELEM; + break; + case LFT_ICAP_REP_HEADER: + lt->type = LFT_ICAP_REP_HEADER_ELEM; + break; +#endif + default: + break; + } + } + + lt->data.header.header = header; + } else { + switch (lt->type) { + case LFT_REQUEST_HEADER: + lt->type = LFT_REQUEST_ALL_HEADERS; + break; + + case LFT_ADAPTED_REQUEST_HEADER: + lt->type = LFT_ADAPTED_REQUEST_ALL_HEADERS; + break; + + case LFT_REPLY_HEADER: + lt->type = LFT_REPLY_ALL_HEADERS; + break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER: + lt->type = LFT_ICAP_LAST_MATCHED_ALL_HEADERS; + break; + case LFT_ICAP_REQ_HEADER: + lt->type = LFT_ICAP_REQ_ALL_HEADERS; + break; + case LFT_ICAP_REP_HEADER: + lt->type = LFT_ICAP_REP_ALL_HEADERS; + break; +#endif + default: + break; + } + Config.onoff.log_mime_hdrs = 1; + } + + break; + + case LFT_CLIENT_FQDN: + Config.onoff.log_fqdn = 1; + break; + + case LFT_TIME_SUBSECOND: + lt->divisor = 1000; + + if (lt->precision) { + int i; + lt->divisor = 1000000; + + for (i = lt->precision; i > 1; i--) + lt->divisor /= 10; + + if (!lt->divisor) + lt->divisor = 0; + } + + break; + + case LFT_HTTP_SENT_STATUS_CODE_OLD_30: + debugs(46, 0, "WARNING: the \"Hs\" formating code is deprecated use the \">Hs\" instead"); + lt->type = LFT_HTTP_SENT_STATUS_CODE; + break; + default: + break; + } + + return (cur - def); +} + +int +accessLogParseLogFormat(logformat_token ** fmt, char *def) +{ + char *cur, *eos; + logformat_token *new_lt, *last_lt; + enum log_quote quote = LOG_QUOTE_NONE; + + debugs(46, 2, "accessLogParseLogFormat: got definition '" << def << "'"); + + /* very inefficent parser, but who cares, this needs to be simple */ + /* First off, let's tokenize, we'll optimize in a second pass. + * A token can either be a %-prefixed sequence (usually a dynamic + * token but it can be an escaped sequence), or a string. */ + cur = def; + eos = def + strlen(def); + *fmt = new_lt = last_lt = (logformat_token *)xmalloc(sizeof(logformat_token)); + cur += accessLogGetNewLogFormatToken(new_lt, cur, "e); + + while (cur < eos) { + new_lt = (logformat_token *)xmalloc(sizeof(logformat_token)); + last_lt->next = new_lt; + last_lt = new_lt; + cur += accessLogGetNewLogFormatToken(new_lt, cur, "e); + } + + return 1; +} + +void +accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definitions) +{ + logformat_token *t; + logformat *format; + + struct logformat_token_table_entry *te; + debugs(46, 4, "accessLogDumpLogFormat called"); + + for (format = definitions; format; format = format->next) { + debugs(46, 3, "Dumping logformat definition for " << format->name); + storeAppendPrintf(entry, "logformat %s ", format->name); + + for (t = format->format; t; t = t->next) { + if (t->type == LFT_STRING) + storeAppendPrintf(entry, "%s", t->data.string); + else { + char argbuf[256]; + char *arg = NULL; + logformat_bcode_t type = t->type; + + switch (type) { + /* special cases */ + + case LFT_STRING: + break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: + case LFT_ICAP_REQ_HEADER_ELEM: + case LFT_ICAP_REP_HEADER_ELEM: +#endif + case LFT_REQUEST_HEADER_ELEM: + case LFT_ADAPTED_REQUEST_HEADER_ELEM: + case LFT_REPLY_HEADER_ELEM: + + if (t->data.header.separator != ',') + snprintf(argbuf, sizeof(argbuf), "%s:%c%s", t->data.header.header, t->data.header.separator, t->data.header.element); + else + snprintf(argbuf, sizeof(argbuf), "%s:%s", t->data.header.header, t->data.header.element); + + arg = argbuf; + + switch (type) { + case LFT_REQUEST_HEADER_ELEM: + type = LFT_REQUEST_HEADER_ELEM; + break; + case LFT_ADAPTED_REQUEST_HEADER_ELEM: + type = LFT_ADAPTED_REQUEST_HEADER_ELEM; + break; + case LFT_REPLY_HEADER_ELEM: + type = LFT_REPLY_HEADER_ELEM; + break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: + type = LFT_ICAP_LAST_MATCHED_HEADER; + break; + case LFT_ICAP_REQ_HEADER_ELEM: + type = LFT_ICAP_REQ_HEADER; + break; + case LFT_ICAP_REP_HEADER_ELEM: + type = LFT_ICAP_REP_HEADER; + break; +#endif + default: + break; + } + + break; + + case LFT_REQUEST_ALL_HEADERS: + case LFT_ADAPTED_REQUEST_ALL_HEADERS: + case LFT_REPLY_ALL_HEADERS: + +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: + case LFT_ICAP_REQ_ALL_HEADERS: + case LFT_ICAP_REP_ALL_HEADERS: +#endif + + switch (type) { + case LFT_REQUEST_ALL_HEADERS: + type = LFT_REQUEST_HEADER; + break; + case LFT_ADAPTED_REQUEST_ALL_HEADERS: + type = LFT_ADAPTED_REQUEST_HEADER; + break; + case LFT_REPLY_ALL_HEADERS: + type = LFT_REPLY_HEADER; + break; +#if ICAP_CLIENT + case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: + type = LFT_ICAP_LAST_MATCHED_HEADER; + break; + case LFT_ICAP_REQ_ALL_HEADERS: + type = LFT_ICAP_REQ_HEADER; + break; + case LFT_ICAP_REP_ALL_HEADERS: + type = LFT_ICAP_REP_HEADER; + break; +#endif + default: + break; + } + + break; + + default: + if (t->data.string) + arg = t->data.string; + + break; + } + + entry->append("%", 1); + + switch (t->quote) { + + case LOG_QUOTE_QUOTES: + entry->append("\"", 1); + break; + + case LOG_QUOTE_MIMEBLOB: + entry->append("[", 1); + break; + + case LOG_QUOTE_URL: + entry->append("#", 1); + break; + + case LOG_QUOTE_RAW: + entry->append("'", 1); + break; + + case LOG_QUOTE_NONE: + break; + } + + if (t->left) + entry->append("-", 1); + + if (t->zero) + entry->append("0", 1); + + if (t->width) + storeAppendPrintf(entry, "%d", (int) t->width); + + if (t->precision) + storeAppendPrintf(entry, ".%d", (int) t->precision); + + if (arg) + storeAppendPrintf(entry, "{%s}", arg); + + for (te = logformat_token_table; te->config != NULL; te++) { + if (te->token_type == type) { + storeAppendPrintf(entry, "%s", te->config); + break; + } + } + + if (t->space) + entry->append(" ", 1); + + assert(te->config != NULL); + } + } + + entry->append("\n", 1); + } + +} + +void +accessLogFreeLogFormat(logformat_token ** tokens) +{ + while (*tokens) { + logformat_token *token = *tokens; + *tokens = token->next; + safe_free(token->data.string); + xfree(token); + } +} + +logformat::logformat(const char *n) : + format(NULL), + next(NULL) +{ + name = xstrdup(n); +} + +logformat::~logformat() +{ + // erase the list without consuming stack space + while (next) { + // unlink the next entry for deletion + logformat *temp = next; + next = temp->next; + temp->next = NULL; + delete temp; + } + + // remove locals + xfree(name); + accessLogFreeLogFormat(&format); +} diff --git a/src/log/Tokens.h b/src/log/Tokens.h new file mode 100644 index 0000000000..36316d454e --- /dev/null +++ b/src/log/Tokens.h @@ -0,0 +1,246 @@ +/* + * $Id$ + * + * DEBUG: section 46 Access Log + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ +#ifndef _SQUID_LOG_TOKENS_H +#define _SQUID_LOG_TOKENS_H + +class StoreEntry; + +#define LOG_BUF_SZ (MAX_URL<<2) + +/* + * Bytecodes for the configureable logformat stuff + */ +typedef enum { + LFT_NONE, /* dummy */ + LFT_STRING, + + LFT_CLIENT_IP_ADDRESS, + LFT_CLIENT_FQDN, + LFT_CLIENT_PORT, +#if USE_SQUID_EUI + LFT_CLIENT_EUI, +#endif + + /*LFT_SERVER_IP_ADDRESS, */ + LFT_SERVER_IP_OR_PEER_NAME, + /*LFT_SERVER_PORT, */ + + LFT_LOCAL_IP, + LFT_LOCAL_PORT, + /*LFT_LOCAL_NAME, */ + LFT_PEER_LOCAL_PORT, + + LFT_TIME_SECONDS_SINCE_EPOCH, + LFT_TIME_SUBSECOND, + LFT_TIME_LOCALTIME, + LFT_TIME_GMT, + LFT_TIME_TO_HANDLE_REQUEST, + + LFT_PEER_RESPONSE_TIME, + LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, + LFT_DNS_WAIT_TIME, + + LFT_REQUEST_HEADER, + LFT_REQUEST_HEADER_ELEM, + LFT_REQUEST_ALL_HEADERS, + + LFT_ADAPTED_REQUEST_HEADER, + LFT_ADAPTED_REQUEST_HEADER_ELEM, + LFT_ADAPTED_REQUEST_ALL_HEADERS, + + LFT_REPLY_HEADER, + LFT_REPLY_HEADER_ELEM, + LFT_REPLY_ALL_HEADERS, + + LFT_USER_NAME, + LFT_USER_LOGIN, + LFT_USER_IDENT, + /*LFT_USER_REALM, */ + /*LFT_USER_SCHEME, */ + LFT_USER_EXTERNAL, + + LFT_HTTP_SENT_STATUS_CODE_OLD_30, + LFT_HTTP_SENT_STATUS_CODE, + LFT_HTTP_RECEIVED_STATUS_CODE, + /*LFT_HTTP_STATUS, */ + LFT_HTTP_BODY_BYTES_READ, + + LFT_SQUID_STATUS, + LFT_SQUID_ERROR, + LFT_SQUID_ERROR_DETAIL, + LFT_SQUID_HIERARCHY, + + LFT_MIME_TYPE, + + LFT_REQUEST_METHOD, + LFT_REQUEST_URI, + LFT_REQUEST_URLPATH, + /*LFT_REQUEST_QUERY, * // * this is not needed. see strip_query_terms */ + LFT_REQUEST_VERSION, + + LFT_REQUEST_SIZE_TOTAL, + /*LFT_REQUEST_SIZE_LINE, */ + LFT_REQUEST_SIZE_HEADERS, + /*LFT_REQUEST_SIZE_BODY, */ + /*LFT_REQUEST_SIZE_BODY_NO_TE, */ + + LFT_REPLY_SIZE_TOTAL, + LFT_REPLY_HIGHOFFSET, + LFT_REPLY_OBJECTSIZE, + /*LFT_REPLY_SIZE_LINE, */ + LFT_REPLY_SIZE_HEADERS, + /*LFT_REPLY_SIZE_BODY, */ + /*LFT_REPLY_SIZE_BODY_NO_TE, */ + + LFT_TAG, + LFT_IO_SIZE_TOTAL, + LFT_EXT_LOG, + + LFT_SEQUENCE_NUMBER, + +#if USE_ADAPTATION + LTF_ADAPTATION_SUM_XACT_TIMES, + LTF_ADAPTATION_ALL_XACT_TIMES, +#endif + +#if ICAP_CLIENT + + LFT_ICAP_TOTAL_TIME, + LFT_ICAP_LAST_MATCHED_HEADER, + LFT_ICAP_LAST_MATCHED_HEADER_ELEM, + LFT_ICAP_LAST_MATCHED_ALL_HEADERS, + + LFT_ICAP_ADDR, + LFT_ICAP_SERV_NAME, + LFT_ICAP_REQUEST_URI, + LFT_ICAP_REQUEST_METHOD, + LFT_ICAP_BYTES_SENT, + LFT_ICAP_BYTES_READ, + LFT_ICAP_BODY_BYTES_READ, + + LFT_ICAP_REQ_HEADER, + LFT_ICAP_REQ_HEADER_ELEM, + LFT_ICAP_REQ_ALL_HEADERS, + + LFT_ICAP_REP_HEADER, + LFT_ICAP_REP_HEADER_ELEM, + LFT_ICAP_REP_ALL_HEADERS, + + LFT_ICAP_TR_RESPONSE_TIME, + LFT_ICAP_IO_TIME, + LFT_ICAP_OUTCOME, + LFT_ICAP_STATUS_CODE, +#endif + + LFT_PERCENT /* special string cases for escaped chars */ +} logformat_bcode_t; + +enum log_quote { + LOG_QUOTE_NONE = 0, + LOG_QUOTE_QUOTES, + LOG_QUOTE_MIMEBLOB, + LOG_QUOTE_URL, + LOG_QUOTE_RAW +}; + +/* FIXME: public class so we can pre-define its type. */ +class logformat_token +{ +public: + logformat_bcode_t type; + union { + char *string; + + struct { + char *header; + char *element; + char separator; + } header; + char *timespec; + } data; + unsigned char width; + unsigned char precision; + enum log_quote quote; + unsigned int left:1; + unsigned int space:1; + unsigned int zero:1; + int divisor; + logformat_token *next; /* todo: move from linked list to array */ +}; + +struct logformat_token_table_entry { + const char *config; + logformat_bcode_t token_type; + int options; +}; + +class logformat +{ +public: + logformat(const char *name); + ~logformat(); + + char *name; + logformat_token *format; + logformat *next; +}; + +extern const char *log_tags[]; +extern struct logformat_token_table_entry logformat_token_table[]; + +#if USE_ADAPTATION +extern bool alLogformatHasAdaptToken; +#endif + +#if ICAP_CLIENT +extern bool alLogformatHasIcapToken; +#endif + +/* parses a single token. Returns the token length in characters, + * and fills in the lt item with the token information. + * def is for sure null-terminated + */ +int accessLogGetNewLogFormatToken(logformat_token * lt, char *def, enum log_quote *quote); + +/* very inefficent parser, but who cares, this needs to be simple */ +/* First off, let's tokenize, we'll optimize in a second pass. + * A token can either be a %-prefixed sequence (usually a dynamic + * token but it can be an escaped sequence), or a string. */ +int accessLogParseLogFormat(logformat_token ** fmt, char *def); + +void accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definitions); + +void accessLogFreeLogFormat(logformat_token ** tokens); + +#endif /* _SQUID_LOG_TOKENS_H */ diff --git a/src/log/access_log.cc b/src/log/access_log.cc index da729f6a0b..8d73765a39 100644 --- a/src/log/access_log.cc +++ b/src/log/access_log.cc @@ -51,14 +51,14 @@ #include "HttpReply.h" #include "HttpRequest.h" #include "log/File.h" +#include "log/Formats.h" +#include "log/Gadgets.h" +#include "log/Tokens.h" #include "MemBuf.h" #include "mgr/Registration.h" #include "rfc1738.h" #include "SquidTime.h" -static void accessLogSquid(AccessLogEntry * al, Logfile * logfile); -static void accessLogCommon(AccessLogEntry * al, Logfile * logfile); -static void accessLogCustom(AccessLogEntry * al, customlog * log); #if HEADERS_LOG static Logfile *headerslog = NULL; #endif @@ -70,1919 +70,25 @@ static struct sockaddr_in mcast_miss_to; static void mcast_encode(unsigned int *, size_t, const unsigned int *); #endif -const char *log_tags[] = { - "NONE", - "TCP_HIT", - "TCP_MISS", - "TCP_REFRESH_UNMODIFIED", - "TCP_REFRESH_FAIL", // same tag logged for LOG_TCP_REFRESH_FAIL_OLD and - "TCP_REFRESH_FAIL", // LOG_TCP_REFRESH_FAIL_ERR for backward-compatibility - "TCP_REFRESH_MODIFIED", - "TCP_CLIENT_REFRESH_MISS", - "TCP_IMS_HIT", - "TCP_SWAPFAIL_MISS", - "TCP_NEGATIVE_HIT", - "TCP_MEM_HIT", - "TCP_DENIED", - "TCP_DENIED_REPLY", - "TCP_OFFLINE_HIT", -#if LOG_TCP_REDIRECTS - "TCP_REDIRECT", -#endif - "UDP_HIT", - "UDP_MISS", - "UDP_DENIED", - "UDP_INVALID", - "UDP_MISS_NOFETCH", - "ICP_QUERY", - "LOG_TYPE_MAX" -}; - -#if USE_FORW_VIA_DB - -typedef struct { - hash_link hash; - int n; -} fvdb_entry; -static hash_table *via_table = NULL; -static hash_table *forw_table = NULL; -static void fvdbInit(); -static void fvdbDumpTable(StoreEntry * e, hash_table * hash); -static void fvdbCount(hash_table * hash, const char *key); -static OBJH fvdbDumpVia; -static OBJH fvdbDumpForw; -static FREE fvdbFreeEntry; -static void fvdbClear(void); -static void fvdbRegisterWithCacheManager(); -#endif - -int LogfileStatus = LOG_DISABLE; - -#if USE_ADAPTATION -bool alLogformatHasAdaptToken = false; -#endif - -#if ICAP_CLIENT -bool alLogformatHasIcapToken = false; -#endif - -#define LOG_BUF_SZ (MAX_URL<<2) - -static const char c2x[] = - "000102030405060708090a0b0c0d0e0f" - "101112131415161718191a1b1c1d1e1f" - "202122232425262728292a2b2c2d2e2f" - "303132333435363738393a3b3c3d3e3f" - "404142434445464748494a4b4c4d4e4f" - "505152535455565758595a5b5c5d5e5f" - "606162636465666768696a6b6c6d6e6f" - "707172737475767778797a7b7c7d7e7f" - "808182838485868788898a8b8c8d8e8f" - "909192939495969798999a9b9c9d9e9f" - "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" - "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" - "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" - "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" - "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" - "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; - -/* log_quote -- URL-style encoding on MIME headers. */ - -char * -log_quote(const char *header) -{ - int c; - int i; - char *buf; - char *buf_cursor; - - if (header == NULL) { - buf = static_cast(xcalloc(1, 1)); - *buf = '\0'; - return buf; - } - - buf = static_cast(xcalloc(1, (strlen(header) * 3) + 1)); - buf_cursor = buf; - /* - * We escape: \x00-\x1F"#%;<>?{}|\\\\^~`\[\]\x7F-\xFF - * which is the default escape list for the CPAN Perl5 URI module - * modulo the inclusion of space (x40) to make the raw logs a bit - * more readable. - */ - - while ((c = *(const unsigned char *) header++) != '\0') { -#if !OLD_LOG_MIME - - if (c == '\r') { - *buf_cursor++ = '\\'; - *buf_cursor++ = 'r'; - } else if (c == '\n') { - *buf_cursor++ = '\\'; - *buf_cursor++ = 'n'; - } else -#endif - if (c <= 0x1F - || c >= 0x7F - || c == '%' -#if OLD_LOG_MIME - || c == '"' - || c == '#' - || c == ';' - || c == '<' - || c == '>' - || c == '?' - || c == '{' - || c == '}' - || c == '|' - || c == '\\' - || c == '^' - || c == '~' - || c == '`' -#endif - || c == '[' - || c == ']') { - *buf_cursor++ = '%'; - i = c * 2; - *buf_cursor++ = c2x[i]; - *buf_cursor++ = c2x[i + 1]; -#if !OLD_LOG_MIME - - } else if (c == '\\') { - *buf_cursor++ = '\\'; - *buf_cursor++ = '\\'; -#endif - - } else { - *buf_cursor++ = (char) c; - } - } - - *buf_cursor = '\0'; - return buf; -} - -static char * -username_quote(const char *header) -/* copy of log_quote. Bugs there will be found here */ -{ - int c; - int i; - char *buf; - char *buf_cursor; - - if (header == NULL) { - buf = static_cast(xcalloc(1, 1)); - *buf = '\0'; - return buf; - } - - buf = static_cast(xcalloc(1, (strlen(header) * 3) + 1)); - buf_cursor = buf; - /* - * We escape: space \x00-\x1F and space (0x40) and \x7F-\xFF - * to prevent garbage in the logs. CR and LF are also there just in case. - */ - - while ((c = *(const unsigned char *) header++) != '\0') { - if (c == '\r') { - *buf_cursor++ = '\\'; - *buf_cursor++ = 'r'; - } else if (c == '\n') { - *buf_cursor++ = '\\'; - *buf_cursor++ = 'n'; - } else if (c <= 0x1F - || c >= 0x7F - || c == '%' - || c == ' ') { - *buf_cursor++ = '%'; - i = c * 2; - *buf_cursor++ = c2x[i]; - *buf_cursor++ = c2x[i + 1]; - } else { - *buf_cursor++ = (char) c; - } - } - - *buf_cursor = '\0'; - return buf; -} - -static char * -accessLogFormatName(const char *name) -{ - if (NULL == name) - return NULL; - - if (name[0] == '\0') - return NULL; - - return username_quote(name); -} - -static char * -log_quoted_string(const char *str) -{ - char *out = (char *)xmalloc(strlen(str) * 2 + 1); - char *p = out; - - while (*str) { - int l = strcspn(str, "\"\\\r\n\t"); - memcpy(p, str, l); - str += l; - p += l; - - switch (*str) { - - case '\0': - break; - - case '\r': - *p++ = '\\'; - *p++ = 'r'; - str++; - break; - - case '\n': - *p++ = '\\'; - *p++ = 'n'; - str++; - break; - - case '\t': - *p++ = '\\'; - *p++ = 't'; - str++; - break; - - default: - *p++ = '\\'; - *p++ = *str; - str++; - break; - } - } - - *p++ = '\0'; - return out; -} - -/* - * Bytecodes for the configureable logformat stuff - */ -typedef enum { - LFT_NONE, /* dummy */ - LFT_STRING, - - LFT_CLIENT_IP_ADDRESS, - LFT_CLIENT_FQDN, - LFT_CLIENT_PORT, -#if USE_SQUID_EUI - LFT_CLIENT_EUI, -#endif - - /*LFT_SERVER_IP_ADDRESS, */ - LFT_SERVER_IP_OR_PEER_NAME, - /*LFT_SERVER_PORT, */ - - LFT_LOCAL_IP, - LFT_LOCAL_PORT, - /*LFT_LOCAL_NAME, */ - LFT_PEER_LOCAL_PORT, - - LFT_TIME_SECONDS_SINCE_EPOCH, - LFT_TIME_SUBSECOND, - LFT_TIME_LOCALTIME, - LFT_TIME_GMT, - LFT_TIME_TO_HANDLE_REQUEST, - - LFT_PEER_RESPONSE_TIME, - LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, - LFT_DNS_WAIT_TIME, - - LFT_REQUEST_HEADER, - LFT_REQUEST_HEADER_ELEM, - LFT_REQUEST_ALL_HEADERS, - - LFT_ADAPTED_REQUEST_HEADER, - LFT_ADAPTED_REQUEST_HEADER_ELEM, - LFT_ADAPTED_REQUEST_ALL_HEADERS, - - LFT_REPLY_HEADER, - LFT_REPLY_HEADER_ELEM, - LFT_REPLY_ALL_HEADERS, - - LFT_USER_NAME, - LFT_USER_LOGIN, - LFT_USER_IDENT, - /*LFT_USER_REALM, */ - /*LFT_USER_SCHEME, */ - LFT_USER_EXTERNAL, - - LFT_HTTP_SENT_STATUS_CODE_OLD_30, - LFT_HTTP_SENT_STATUS_CODE, - LFT_HTTP_RECEIVED_STATUS_CODE, - /*LFT_HTTP_STATUS, */ - LFT_HTTP_BODY_BYTES_READ, - - LFT_SQUID_STATUS, - LFT_SQUID_ERROR, - LFT_SQUID_ERROR_DETAIL, - LFT_SQUID_HIERARCHY, - - LFT_MIME_TYPE, - - LFT_REQUEST_METHOD, - LFT_REQUEST_URI, - LFT_REQUEST_URLPATH, - /*LFT_REQUEST_QUERY, * // * this is not needed. see strip_query_terms */ - LFT_REQUEST_VERSION, - - LFT_REQUEST_SIZE_TOTAL, - /*LFT_REQUEST_SIZE_LINE, */ - LFT_REQUEST_SIZE_HEADERS, - /*LFT_REQUEST_SIZE_BODY, */ - /*LFT_REQUEST_SIZE_BODY_NO_TE, */ - - LFT_REPLY_SIZE_TOTAL, - LFT_REPLY_HIGHOFFSET, - LFT_REPLY_OBJECTSIZE, - /*LFT_REPLY_SIZE_LINE, */ - LFT_REPLY_SIZE_HEADERS, - /*LFT_REPLY_SIZE_BODY, */ - /*LFT_REPLY_SIZE_BODY_NO_TE, */ - - LFT_TAG, - LFT_IO_SIZE_TOTAL, - LFT_EXT_LOG, - - LFT_SEQUENCE_NUMBER, - -#if USE_ADAPTATION - LTF_ADAPTATION_SUM_XACT_TIMES, - LTF_ADAPTATION_ALL_XACT_TIMES, -#endif - -#if ICAP_CLIENT - - LFT_ICAP_TOTAL_TIME, - LFT_ICAP_LAST_MATCHED_HEADER, - LFT_ICAP_LAST_MATCHED_HEADER_ELEM, - LFT_ICAP_LAST_MATCHED_ALL_HEADERS, - - LFT_ICAP_ADDR, - LFT_ICAP_SERV_NAME, - LFT_ICAP_REQUEST_URI, - LFT_ICAP_REQUEST_METHOD, - LFT_ICAP_BYTES_SENT, - LFT_ICAP_BYTES_READ, - LFT_ICAP_BODY_BYTES_READ, - - LFT_ICAP_REQ_HEADER, - LFT_ICAP_REQ_HEADER_ELEM, - LFT_ICAP_REQ_ALL_HEADERS, - - LFT_ICAP_REP_HEADER, - LFT_ICAP_REP_HEADER_ELEM, - LFT_ICAP_REP_ALL_HEADERS, - - LFT_ICAP_TR_RESPONSE_TIME, - LFT_ICAP_IO_TIME, - LFT_ICAP_OUTCOME, - LFT_ICAP_STATUS_CODE, -#endif - - LFT_PERCENT /* special string cases for escaped chars */ -} logformat_bcode_t; - -enum log_quote { - LOG_QUOTE_NONE = 0, - LOG_QUOTE_QUOTES, - LOG_QUOTE_BRAKETS, - LOG_QUOTE_URL, - LOG_QUOTE_RAW -}; - -/* FIXME: public class so we can pre-define its type. */ -class logformat_token -{ -public: - logformat_bcode_t type; - union { - char *string; - - struct { - char *header; - char *element; - char separator; - } header; - char *timespec; - } data; - unsigned char width; - unsigned char precision; - enum log_quote quote; - unsigned int left:1; - unsigned int space:1; - unsigned int zero:1; - int divisor; - logformat_token *next; /* todo: move from linked list to array */ -}; - -struct logformat_token_table_entry { - const char *config; - logformat_bcode_t token_type; - int options; -}; - -struct logformat_token_table_entry logformat_token_table[] = { - - {">a", LFT_CLIENT_IP_ADDRESS}, - {">p", LFT_CLIENT_PORT}, - {">A", LFT_CLIENT_FQDN}, -#if USE_SQUID_EUI - {">eui", LFT_CLIENT_EUI}, -#endif - - /*{ "ha", LFT_ADAPTED_REQUEST_HEADER}, - {">ha", LFT_ADAPTED_REQUEST_ALL_HEADERS}, - {">h", LFT_REQUEST_HEADER}, - {">h", LFT_REQUEST_ALL_HEADERS}, - {"Hs", LFT_HTTP_SENT_STATUS_CODE}, - {"v", LFT_REQUEST_VERSION}, - {"rv", LFT_REQUEST_VERSION}, - - { ">st", LFT_REQUEST_SIZE_TOTAL }, - /*{ ">sl", LFT_REQUEST_SIZE_LINE }, * / / * the request line "GET ... " */ - { ">sh", LFT_REQUEST_SIZE_HEADERS }, - /*{ ">sb", LFT_REQUEST_SIZE_BODY }, */ - /*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */ - - {"st", LFT_ICAP_BYTES_SENT}, - {"icap::h", LFT_ICAP_REQ_HEADER}, - {"icap::logFormat; - logfile = log->logfile; - - for (fmt = lf->format; fmt != NULL; fmt = fmt->next) { /* for each token */ - const char *out = NULL; - int quote = 0; - long int outint = 0; - int doint = 0; - int dofree = 0; - int64_t outoff = 0; - int dooff = 0; - - switch (fmt->type) { - - case LFT_NONE: - out = ""; - break; - - case LFT_STRING: - out = fmt->data.string; - break; - - case LFT_CLIENT_IP_ADDRESS: - if (al->cache.caddr.IsNoAddr()) // e.g., ICAP OPTIONS lack client - out = "-"; - else - out = al->cache.caddr.NtoA(tmp,1024); - break; - - case LFT_CLIENT_FQDN: - if (al->cache.caddr.IsAnyAddr()) // e.g., ICAP OPTIONS lack client - out = "-"; - else - out = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); - if (!out) { - out = al->cache.caddr.NtoA(tmp,1024); - } - - break; - - case LFT_CLIENT_PORT: - if (al->request) { - outint = al->request->client_addr.GetPort(); - doint = 1; - } - break; - -#if USE_SQUID_EUI - case LFT_CLIENT_EUI: - if (al->request) { - if (al->cache.caddr.IsIPv4()) - al->request->client_eui48.encode(tmp, 1024); - else - al->request->client_eui64.encode(tmp, 1024); - out = tmp; - } - break; -#endif - - /* case LFT_SERVER_IP_ADDRESS: */ - - case LFT_SERVER_IP_OR_PEER_NAME: - out = al->hier.host; - - break; - - /* case LFT_SERVER_PORT: */ - - case LFT_LOCAL_IP: - if (al->request) { - out = al->request->my_addr.NtoA(tmp,1024); - } - - break; - - case LFT_LOCAL_PORT: - if (al->request) { - outint = al->request->my_addr.GetPort(); - doint = 1; - } - - break; - - case LFT_PEER_LOCAL_PORT: - if (al->hier.peer_local_port) { - outint = al->hier.peer_local_port; - doint = 1; - } - - break; - - case LFT_TIME_SECONDS_SINCE_EPOCH: - // some platforms store time in 32-bit, some 64-bit... - outoff = static_cast(current_time.tv_sec); - dooff = 1; - break; - - case LFT_TIME_SUBSECOND: - outint = current_time.tv_usec / fmt->divisor; - doint = 1; - break; - - - case LFT_TIME_LOCALTIME: - - case LFT_TIME_GMT: { - const char *spec; - - struct tm *t; - spec = fmt->data.timespec; - - if (fmt->type == LFT_TIME_LOCALTIME) { - if (!spec) - spec = "%d/%b/%Y:%H:%M:%S %z"; - t = localtime(&squid_curtime); - } else { - if (!spec) - spec = "%d/%b/%Y:%H:%M:%S"; - - t = gmtime(&squid_curtime); - } - - strftime(tmp, sizeof(tmp), spec, t); - - out = tmp; - } - - break; - - case LFT_TIME_TO_HANDLE_REQUEST: - outint = al->cache.msec; - doint = 1; - break; - - case LFT_PEER_RESPONSE_TIME: - if (al->hier.peer_response_time < 0) { - out = "-"; - } else { - outoff = al->hier.peer_response_time; - dooff = 1; - } - break; - - case LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME: - if (al->hier.total_response_time < 0) { - out = "-"; - } else { - outoff = al->hier.total_response_time; - dooff = 1; - } - break; - - case LFT_DNS_WAIT_TIME: - if (al->request && al->request->dnsWait >= 0) { - outint = al->request->dnsWait; - doint = 1; - } - break; - - case LFT_REQUEST_HEADER: - - if (al->request) - sb = al->request->header.getByName(fmt->data.header.header); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ADAPTED_REQUEST_HEADER: - - if (al->request) - sb = al->adapted_request->header.getByName(fmt->data.header.header); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_REPLY_HEADER: - if (al->reply) - sb = al->reply->header.getByName(fmt->data.header.header); - - out = sb.termedBuf(); - - quote = 1; - - break; - -#if USE_ADAPTATION - case LTF_ADAPTATION_SUM_XACT_TIMES: - if (al->request) { - Adaptation::History::Pointer ah = al->request->adaptHistory(); - if (ah != NULL) - ah->sumLogString(fmt->data.string, sb); - out = sb.termedBuf(); - } - break; - - case LTF_ADAPTATION_ALL_XACT_TIMES: - if (al->request) { - Adaptation::History::Pointer ah = al->request->adaptHistory(); - if (ah != NULL) - ah->allLogString(fmt->data.string, sb); - out = sb.termedBuf(); - } - break; -#endif - -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER: - if (al->request) { - Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); - if (ih != NULL) - sb = ih->mergeOfIcapHeaders.getByName(fmt->data.header.header); - } - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: - if (al->request) { - Adaptation::Icap::History::Pointer ih = al->request->icapHistory(); - if (ih != NULL) - sb = ih->mergeOfIcapHeaders.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - } - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: - out = al->headers.icap; - - quote = 1; - - break; - - case LFT_ICAP_ADDR: - if (!out) - out = al->icap.hostAddr.NtoA(tmp,1024); - break; - - case LFT_ICAP_SERV_NAME: - out = al->icap.serviceName.termedBuf(); - break; - - case LFT_ICAP_REQUEST_URI: - out = al->icap.reqUri.termedBuf(); - break; - - case LFT_ICAP_REQUEST_METHOD: - out = Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod); - break; - - case LFT_ICAP_BYTES_SENT: - outoff = al->icap.bytesSent; - dooff = 1; - break; - - case LFT_ICAP_BYTES_READ: - outoff = al->icap.bytesRead; - dooff = 1; - break; - - case LFT_ICAP_BODY_BYTES_READ: - if (al->icap.bodyBytesRead >= 0) { - outoff = al->icap.bodyBytesRead; - dooff = 1; - } - // else if icap.bodyBytesRead < 0, we do not have any http data, - // so just print a "-" (204 responses etc) - break; - - case LFT_ICAP_REQ_HEADER: - if (NULL != al->icap.request) { - sb = al->icap.request->header.getByName(fmt->data.header.header); - out = sb.termedBuf(); - quote = 1; - } - break; - - case LFT_ICAP_REQ_HEADER_ELEM: - if (al->request) - sb = al->icap.request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ICAP_REQ_ALL_HEADERS: - if (al->icap.request) { - HttpHeaderPos pos = HttpHeaderInitPos; - while (const HttpHeaderEntry *e = al->icap.request->header.getEntry(&pos)) { - sb.append(e->name); - sb.append(": "); - sb.append(e->value); - sb.append("\r\n"); - } - out = sb.termedBuf(); - quote = 1; - } - break; - - case LFT_ICAP_REP_HEADER: - if (NULL != al->icap.reply) { - sb = al->icap.reply->header.getByName(fmt->data.header.header); - out = sb.termedBuf(); - quote = 1; - } - break; - - case LFT_ICAP_REP_HEADER_ELEM: - if (NULL != al->icap.reply) - sb = al->icap.reply->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ICAP_REP_ALL_HEADERS: - if (al->icap.reply) { - HttpHeaderPos pos = HttpHeaderInitPos; - while (const HttpHeaderEntry *e = al->icap.reply->header.getEntry(&pos)) { - sb.append(e->name); - sb.append(": "); - sb.append(e->value); - sb.append("\r\n"); - } - out = sb.termedBuf(); - quote = 1; - } - break; - - case LFT_ICAP_TR_RESPONSE_TIME: - outint = al->icap.trTime; - doint = 1; - break; - - case LFT_ICAP_IO_TIME: - outint = al->icap.ioTime; - doint = 1; - break; - - case LFT_ICAP_STATUS_CODE: - outint = al->icap.resStatus; - doint = 1; - break; - - case LFT_ICAP_OUTCOME: - out = al->icap.outcome; - break; - - case LFT_ICAP_TOTAL_TIME: - outint = al->icap.processingTime; - doint = 1; - break; -#endif - case LFT_REQUEST_HEADER_ELEM: - if (al->request) - sb = al->request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_ADAPTED_REQUEST_HEADER_ELEM: - if (al->adapted_request) - sb = al->adapted_request->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_REPLY_HEADER_ELEM: - if (al->reply) - sb = al->reply->header.getByNameListMember(fmt->data.header.header, fmt->data.header.element, fmt->data.header.separator); - - out = sb.termedBuf(); - - quote = 1; - - break; - - case LFT_REQUEST_ALL_HEADERS: - out = al->headers.request; - - quote = 1; - - break; - - case LFT_ADAPTED_REQUEST_ALL_HEADERS: - out = al->headers.adapted_request; - - quote = 1; - - break; - - case LFT_REPLY_ALL_HEADERS: - out = al->headers.reply; - - quote = 1; - - break; - - case LFT_USER_NAME: - out = accessLogFormatName(al->cache.authuser); - - if (!out) - out = accessLogFormatName(al->cache.extuser); - -#if USE_SSL - - if (!out) - out = accessLogFormatName(al->cache.ssluser); - -#endif - - if (!out) - out = accessLogFormatName(al->cache.rfc931); - - dofree = 1; - - break; - - case LFT_USER_LOGIN: - out = accessLogFormatName(al->cache.authuser); - - dofree = 1; - - break; - - case LFT_USER_IDENT: - out = accessLogFormatName(al->cache.rfc931); - - dofree = 1; - - break; - - case LFT_USER_EXTERNAL: - out = accessLogFormatName(al->cache.extuser); - - dofree = 1; - - break; - - /* case LFT_USER_REALM: */ - /* case LFT_USER_SCHEME: */ - - // the fmt->type can not be LFT_HTTP_SENT_STATUS_CODE_OLD_30 - // but compiler complains if ommited - case LFT_HTTP_SENT_STATUS_CODE_OLD_30: - case LFT_HTTP_SENT_STATUS_CODE: - outint = al->http.code; - - doint = 1; - - break; - - case LFT_HTTP_RECEIVED_STATUS_CODE: - if (al->hier.peer_reply_status == HTTP_STATUS_NONE) { - out = "-"; - } else { - outint = al->hier.peer_reply_status; - doint = 1; - } - break; - /* case LFT_HTTP_STATUS: - * out = statusline->text; - * quote = 1; - * break; - */ - case LFT_HTTP_BODY_BYTES_READ: - if (al->hier.bodyBytesRead >= 0) { - outoff = al->hier.bodyBytesRead; - dooff = 1; - } - // else if hier.bodyBytesRead < 0 we did not have any data exchange with - // a peer server so just print a "-" (eg requests served from cache, - // or internal error messages). - break; - - case LFT_SQUID_STATUS: - if (al->http.timedout || al->http.aborted) { - snprintf(tmp, sizeof(tmp), "%s%s", log_tags[al->cache.code], - al->http.statusSfx()); - out = tmp; - } else { - out = log_tags[al->cache.code]; - } - - break; - - case LFT_SQUID_ERROR: - if (al->request && al->request->errType != ERR_NONE) - out = errorPageName(al->request->errType); - break; - - case LFT_SQUID_ERROR_DETAIL: - if (al->request && al->request->errDetail != ERR_DETAIL_NONE) { - if (al->request->errDetail > ERR_DETAIL_START && - al->request->errDetail < ERR_DETAIL_MAX) - out = errorDetailName(al->request->errDetail); - else { - if (al->request->errDetail >= ERR_DETAIL_EXCEPTION_START) - snprintf(tmp, sizeof(tmp), "%s=0x%X", - errorDetailName(al->request->errDetail), (uint32_t) al->request->errDetail); - else - snprintf(tmp, sizeof(tmp), "%s=%d", - errorDetailName(al->request->errDetail), al->request->errDetail); - out = tmp; - } - } - break; - - case LFT_SQUID_HIERARCHY: - if (al->hier.ping.timedout) - mb.append("TIMEOUT_", 8); - - out = hier_code_str[al->hier.code]; - - break; - - case LFT_MIME_TYPE: - out = al->http.content_type; - - break; - - case LFT_REQUEST_METHOD: - out = al->_private.method_str; - - break; - - case LFT_REQUEST_URI: - out = al->url; - - break; - - case LFT_REQUEST_URLPATH: - if (al->request) { - out = al->request->urlpath.termedBuf(); - quote = 1; - } - break; - - case LFT_REQUEST_VERSION: - snprintf(tmp, sizeof(tmp), "%d.%d", (int) al->http.version.major, (int) al->http.version.minor); - out = tmp; - break; - - case LFT_REQUEST_SIZE_TOTAL: - outoff = al->cache.requestSize; - dooff = 1; - break; - - /*case LFT_REQUEST_SIZE_LINE: */ - case LFT_REQUEST_SIZE_HEADERS: - outoff = al->cache.requestHeadersSize; - dooff =1; - break; - /*case LFT_REQUEST_SIZE_BODY: */ - /*case LFT_REQUEST_SIZE_BODY_NO_TE: */ - - case LFT_REPLY_SIZE_TOTAL: - outoff = al->cache.replySize; - dooff = 1; - break; - - case LFT_REPLY_HIGHOFFSET: - outoff = al->cache.highOffset; - - dooff = 1; - - break; - - case LFT_REPLY_OBJECTSIZE: - outoff = al->cache.objectSize; - - dooff = 1; - - break; - - /*case LFT_REPLY_SIZE_LINE: */ - case LFT_REPLY_SIZE_HEADERS: - outint = al->cache.replyHeadersSize; - doint = 1; - break; - /*case LFT_REPLY_SIZE_BODY: */ - /*case LFT_REPLY_SIZE_BODY_NO_TE: */ - - case LFT_TAG: - if (al->request) - out = al->request->tag.termedBuf(); - - quote = 1; - - break; - - case LFT_IO_SIZE_TOTAL: - outint = al->cache.requestSize + al->cache.replySize; - doint = 1; - break; - - case LFT_EXT_LOG: - if (al->request) - out = al->request->extacl_log.termedBuf(); - - quote = 1; - - break; - - case LFT_SEQUENCE_NUMBER: - outoff = logfile->sequence_number; - dooff = 1; - break; - - case LFT_PERCENT: - out = "%"; - - break; - } - - if (dooff) { - snprintf(tmp, sizeof(tmp), "%0*" PRId64, fmt->zero ? (int) fmt->width : 0, outoff); - out = tmp; - - } else if (doint) { - snprintf(tmp, sizeof(tmp), "%0*ld", fmt->zero ? (int) fmt->width : 0, outint); - out = tmp; - } - - if (out && *out) { - if (quote || fmt->quote != LOG_QUOTE_NONE) { - char *newout = NULL; - int newfree = 0; - - switch (fmt->quote) { - - case LOG_QUOTE_NONE: - newout = rfc1738_escape_unescaped(out); - break; - - case LOG_QUOTE_QUOTES: - newout = log_quoted_string(out); - newfree = 1; - break; - - case LOG_QUOTE_BRAKETS: - newout = log_quote(out); - newfree = 1; - break; - - case LOG_QUOTE_URL: - newout = rfc1738_escape(out); - break; - - case LOG_QUOTE_RAW: - break; - } - - if (newout) { - if (dofree) - safe_free(out); - - out = newout; - - dofree = newfree; - } - } - - if (fmt->width) { - if (fmt->left) - mb.Printf("%-*s", (int) fmt->width, out); - else - mb.Printf("%*s", (int) fmt->width, out); - } else - mb.append(out, strlen(out)); - } else { - mb.append("-", 1); - } - - if (fmt->space) - mb.append(" ", 1); - - sb.clean(); - - if (dofree) - safe_free(out); - } - - logfilePrintf(logfile, "%s\n", mb.buf); -} - -/* parses a single token. Returns the token length in characters, - * and fills in the lt item with the token information. - * def is for sure null-terminated - */ -static int -accessLogGetNewLogFormatToken(logformat_token * lt, char *def, enum log_quote *quote) -{ - char *cur = def; - - struct logformat_token_table_entry *lte; - int l; - - memset(lt, 0, sizeof(*lt)); - l = strcspn(cur, "%"); - - if (l > 0) { - char *cp; - /* it's a string for sure, until \0 or the next % */ - cp = (char *)xmalloc(l + 1); - xstrncpy(cp, cur, l + 1); - lt->type = LFT_STRING; - lt->data.string = cp; - - while (l > 0) { - switch (*cur) { - - case '"': - - if (*quote == LOG_QUOTE_NONE) - *quote = LOG_QUOTE_QUOTES; - else if (*quote == LOG_QUOTE_QUOTES) - *quote = LOG_QUOTE_NONE; - - break; - - case '[': - if (*quote == LOG_QUOTE_NONE) - *quote = LOG_QUOTE_BRAKETS; - - break; - - case ']': - if (*quote == LOG_QUOTE_BRAKETS) - *quote = LOG_QUOTE_NONE; - - break; - } - - cur++; - l--; - } - - goto done; - } - - if (!*cur) - goto done; - - cur++; - - switch (*cur) { - - case '"': - lt->quote = LOG_QUOTE_QUOTES; - cur++; - break; - - case '\'': - lt->quote = LOG_QUOTE_RAW; - cur++; - break; - - case '[': - lt->quote = LOG_QUOTE_BRAKETS; - cur++; - break; - - case '#': - lt->quote = LOG_QUOTE_URL; - cur++; - break; - - default: - lt->quote = *quote; - break; - } - - if (*cur == '-') { - lt->left = 1; - cur++; - } - - if (*cur == '0') { - lt->zero = 1; - cur++; - } - - if (xisdigit(*cur)) - lt->width = strtol(cur, &cur, 10); - - if (*cur == '.') - lt->precision = strtol(cur + 1, &cur, 10); - - if (*cur == '{') { - char *cp; - cur++; - l = strcspn(cur, "}"); - cp = (char *)xmalloc(l + 1); - xstrncpy(cp, cur, l + 1); - lt->data.string = cp; - cur += l; - - if (*cur == '}') - cur++; - } - - // For upward compatibility, assume "http::" prefix as default prefix - // for all log access formating codes, except those starting - // from "icap::", "adapt::" and "%" - if (strncmp(cur,"http::", 6) == 0 && - strncmp(cur+6, "icap::", 6) != 0 && - strncmp(cur+6, "adapt::", 12) != 0 && *(cur+6) != '%' ) { - cur += 6; - } - - lt->type = LFT_NONE; - - for (lte = logformat_token_table; lte->config != NULL; lte++) { - if (strncmp(lte->config, cur, strlen(lte->config)) == 0) { - lt->type = lte->token_type; - cur += strlen(lte->config); - break; - } - } - - if (lt->type == LFT_NONE) { - fatalf("Can't parse configuration token: '%s'\n", - def); - } - - if (*cur == ' ') { - lt->space = 1; - cur++; - } - -done: - - switch (lt->type) { - -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER: - - case LFT_ICAP_REQ_HEADER: - - case LFT_ICAP_REP_HEADER: -#endif - - case LFT_ADAPTED_REQUEST_HEADER: - - case LFT_REQUEST_HEADER: - - case LFT_REPLY_HEADER: - - if (lt->data.string) { - char *header = lt->data.string; - char *cp = strchr(header, ':'); - - if (cp) { - *cp++ = '\0'; - - if (*cp == ',' || *cp == ';' || *cp == ':') - lt->data.header.separator = *cp++; - else - lt->data.header.separator = ','; - - lt->data.header.element = cp; - - switch (lt->type) { - case LFT_REQUEST_HEADER: - lt->type = LFT_REQUEST_HEADER_ELEM; - break; - - case LFT_ADAPTED_REQUEST_HEADER: - lt->type = LFT_ADAPTED_REQUEST_HEADER_ELEM; - break; - - case LFT_REPLY_HEADER: - lt->type = LFT_REPLY_HEADER_ELEM; - break; -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER: - lt->type = LFT_ICAP_LAST_MATCHED_HEADER_ELEM; - break; - case LFT_ICAP_REQ_HEADER: - lt->type = LFT_ICAP_REQ_HEADER_ELEM; - break; - case LFT_ICAP_REP_HEADER: - lt->type = LFT_ICAP_REP_HEADER_ELEM; - break; -#endif - default: - break; - } - } - - lt->data.header.header = header; - } else { - switch (lt->type) { - case LFT_REQUEST_HEADER: - lt->type = LFT_REQUEST_ALL_HEADERS; - break; - - case LFT_ADAPTED_REQUEST_HEADER: - lt->type = LFT_ADAPTED_REQUEST_ALL_HEADERS; - break; - - case LFT_REPLY_HEADER: - lt->type = LFT_REPLY_ALL_HEADERS; - break; -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER: - lt->type = LFT_ICAP_LAST_MATCHED_ALL_HEADERS; - break; - case LFT_ICAP_REQ_HEADER: - lt->type = LFT_ICAP_REQ_ALL_HEADERS; - break; - case LFT_ICAP_REP_HEADER: - lt->type = LFT_ICAP_REP_ALL_HEADERS; - break; -#endif - default: - break; - } - Config.onoff.log_mime_hdrs = 1; - } - - break; - - case LFT_CLIENT_FQDN: - Config.onoff.log_fqdn = 1; - break; - - case LFT_TIME_SUBSECOND: - lt->divisor = 1000; - - if (lt->precision) { - int i; - lt->divisor = 1000000; - - for (i = lt->precision; i > 1; i--) - lt->divisor /= 10; - - if (!lt->divisor) - lt->divisor = 0; - } - - break; - - case LFT_HTTP_SENT_STATUS_CODE_OLD_30: - debugs(46, 0, "WARNING: the \"Hs\" formating code is deprecated use the \">Hs\" instead"); - lt->type = LFT_HTTP_SENT_STATUS_CODE; - break; - default: - break; - } - - return (cur - def); -} - -int -accessLogParseLogFormat(logformat_token ** fmt, char *def) -{ - char *cur, *eos; - logformat_token *new_lt, *last_lt; - enum log_quote quote = LOG_QUOTE_NONE; - - debugs(46, 2, "accessLogParseLogFormat: got definition '" << def << "'"); - - /* very inefficent parser, but who cares, this needs to be simple */ - /* First off, let's tokenize, we'll optimize in a second pass. - * A token can either be a %-prefixed sequence (usually a dynamic - * token but it can be an escaped sequence), or a string. */ - cur = def; - eos = def + strlen(def); - *fmt = new_lt = last_lt = (logformat_token *)xmalloc(sizeof(logformat_token)); - cur += accessLogGetNewLogFormatToken(new_lt, cur, "e); - - while (cur < eos) { - new_lt = (logformat_token *)xmalloc(sizeof(logformat_token)); - last_lt->next = new_lt; - last_lt = new_lt; - cur += accessLogGetNewLogFormatToken(new_lt, cur, "e); - } - - return 1; -} - -void -accessLogDumpLogFormat(StoreEntry * entry, const char *name, logformat * definitions) -{ - logformat_token *t; - logformat *format; - - struct logformat_token_table_entry *te; - debugs(46, 4, "accessLogDumpLogFormat called"); - - for (format = definitions; format; format = format->next) { - debugs(46, 3, "Dumping logformat definition for " << format->name); - storeAppendPrintf(entry, "logformat %s ", format->name); - - for (t = format->format; t; t = t->next) { - if (t->type == LFT_STRING) - storeAppendPrintf(entry, "%s", t->data.string); - else { - char argbuf[256]; - char *arg = NULL; - logformat_bcode_t type = t->type; - - switch (type) { - /* special cases */ - - case LFT_STRING: - break; -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: - case LFT_ICAP_REQ_HEADER_ELEM: - case LFT_ICAP_REP_HEADER_ELEM: -#endif - case LFT_REQUEST_HEADER_ELEM: - case LFT_ADAPTED_REQUEST_HEADER_ELEM: - case LFT_REPLY_HEADER_ELEM: - - if (t->data.header.separator != ',') - snprintf(argbuf, sizeof(argbuf), "%s:%c%s", t->data.header.header, t->data.header.separator, t->data.header.element); - else - snprintf(argbuf, sizeof(argbuf), "%s:%s", t->data.header.header, t->data.header.element); - - arg = argbuf; - - switch (type) { - case LFT_REQUEST_HEADER_ELEM: - type = LFT_REQUEST_HEADER_ELEM; - break; - case LFT_ADAPTED_REQUEST_HEADER_ELEM: - type = LFT_ADAPTED_REQUEST_HEADER_ELEM; - break; - case LFT_REPLY_HEADER_ELEM: - type = LFT_REPLY_HEADER_ELEM; - break; -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_HEADER_ELEM: - type = LFT_ICAP_LAST_MATCHED_HEADER; - break; - case LFT_ICAP_REQ_HEADER_ELEM: - type = LFT_ICAP_REQ_HEADER; - break; - case LFT_ICAP_REP_HEADER_ELEM: - type = LFT_ICAP_REP_HEADER; - break; -#endif - default: - break; - } - - break; - - case LFT_REQUEST_ALL_HEADERS: - case LFT_ADAPTED_REQUEST_ALL_HEADERS: - case LFT_REPLY_ALL_HEADERS: - -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: - case LFT_ICAP_REQ_ALL_HEADERS: - case LFT_ICAP_REP_ALL_HEADERS: -#endif - - switch (type) { - case LFT_REQUEST_ALL_HEADERS: - type = LFT_REQUEST_HEADER; - break; - case LFT_ADAPTED_REQUEST_ALL_HEADERS: - type = LFT_ADAPTED_REQUEST_HEADER; - break; - case LFT_REPLY_ALL_HEADERS: - type = LFT_REPLY_HEADER; - break; -#if ICAP_CLIENT - case LFT_ICAP_LAST_MATCHED_ALL_HEADERS: - type = LFT_ICAP_LAST_MATCHED_HEADER; - break; - case LFT_ICAP_REQ_ALL_HEADERS: - type = LFT_ICAP_REQ_HEADER; - break; - case LFT_ICAP_REP_ALL_HEADERS: - type = LFT_ICAP_REP_HEADER; - break; -#endif - default: - break; - } - - break; - - default: - if (t->data.string) - arg = t->data.string; - - break; - } - - entry->append("%", 1); - - switch (t->quote) { - - case LOG_QUOTE_QUOTES: - entry->append("\"", 1); - break; - - case LOG_QUOTE_BRAKETS: - entry->append("[", 1); - break; - - case LOG_QUOTE_URL: - entry->append("#", 1); - break; - - case LOG_QUOTE_RAW: - entry->append("'", 1); - break; - - case LOG_QUOTE_NONE: - break; - } - - if (t->left) - entry->append("-", 1); - - if (t->zero) - entry->append("0", 1); - - if (t->width) - storeAppendPrintf(entry, "%d", (int) t->width); - - if (t->precision) - storeAppendPrintf(entry, ".%d", (int) t->precision); - - if (arg) - storeAppendPrintf(entry, "{%s}", arg); - - for (te = logformat_token_table; te->config != NULL; te++) { - if (te->token_type == type) { - storeAppendPrintf(entry, "%s", te->config); - break; - } - } - - if (t->space) - entry->append(" ", 1); - - assert(te->config != NULL); - } - } - - entry->append("\n", 1); - } - -} - -void -accessLogFreeLogFormat(logformat_token ** tokens) -{ - while (*tokens) { - logformat_token *token = *tokens; - *tokens = token->next; - safe_free(token->data.string); - xfree(token); - } -} - -static void -accessLogSquid(AccessLogEntry * al, Logfile * logfile) -{ - const char *client = NULL; - const char *user = NULL; - char buf[MAX_IPSTRLEN]; - - if (Config.onoff.log_fqdn) { - client = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); - } - - if (client == NULL) { - client = al->cache.caddr.NtoA(buf,MAX_IPSTRLEN); - } - - user = accessLogFormatName(al->cache.authuser); - - if (!user) - user = accessLogFormatName(al->cache.extuser); - -#if USE_SSL - - if (!user) - user = accessLogFormatName(al->cache.ssluser); - -#endif - - if (!user) - user = accessLogFormatName(al->cache.rfc931); - - if (user && !*user) - safe_free(user); - - if (!Config.onoff.log_mime_hdrs) { - logfilePrintf(logfile, "%9ld.%03d %6d %s %s%s/%03d %"PRId64" %s %s %s %s%s/%s %s\n", - (long int) current_time.tv_sec, - (int) current_time.tv_usec / 1000, - al->cache.msec, - client, - log_tags[al->cache.code], - al->http.statusSfx(), - al->http.code, - al->cache.replySize, - al->_private.method_str, - al->url, - user ? user : dash_str, - al->hier.ping.timedout ? "TIMEOUT_" : "", - hier_code_str[al->hier.code], - al->hier.host, - al->http.content_type); - } else { - char *ereq = log_quote(al->headers.request); - char *erep = log_quote(al->headers.reply); - logfilePrintf(logfile, "%9ld.%03d %6d %s %s%s/%03d %"PRId64" %s %s %s %s%s/%s %s [%s] [%s]\n", - (long int) current_time.tv_sec, - (int) current_time.tv_usec / 1000, - al->cache.msec, - client, - log_tags[al->cache.code], - al->http.statusSfx(), - al->http.code, - al->cache.replySize, - al->_private.method_str, - al->url, - user ? user : dash_str, - al->hier.ping.timedout ? "TIMEOUT_" : "", - hier_code_str[al->hier.code], - al->hier.host, - al->http.content_type, - ereq, - erep); - safe_free(ereq); - safe_free(erep); - } - safe_free(user); -} - -static void -accessLogCommon(AccessLogEntry * al, Logfile * logfile) -{ - const char *client = NULL; - char *user1 = NULL, *user2 = NULL; - char buf[MAX_IPSTRLEN]; - - if (Config.onoff.log_fqdn) { - client = fqdncache_gethostbyaddr(al->cache.caddr, 0); - } - - if (client == NULL) { - client = al->cache.caddr.NtoA(buf,MAX_IPSTRLEN); - } - - user1 = accessLogFormatName(al->cache.authuser); - - user2 = accessLogFormatName(al->cache.rfc931); - - logfilePrintf(logfile, "%s %s %s [%s] \"%s %s HTTP/%d.%d\" %d %"PRId64" %s%s:%s%s", - client, - user2 ? user2 : dash_str, - user1 ? user1 : dash_str, - mkhttpdlogtime(&squid_curtime), - al->_private.method_str, - al->url, - al->http.version.major, al->http.version.minor, - al->http.code, - al->cache.replySize, - log_tags[al->cache.code], - al->http.statusSfx(), - hier_code_str[al->hier.code], - (Config.onoff.log_mime_hdrs?"":"\n")); - - safe_free(user1); - - safe_free(user2); - - if (Config.onoff.log_mime_hdrs) { - char *ereq = log_quote(al->headers.request); - char *erep = log_quote(al->headers.reply); - logfilePrintf(logfile, " [%s] [%s]\n", ereq, erep); - safe_free(ereq); - safe_free(erep); - } -} - -#if ICAP_CLIENT -static void -accessLogICAPSquid(AccessLogEntry * al, Logfile * logfile) -{ - const char *client = NULL; - const char *user = NULL; - char tmp[MAX_IPSTRLEN], clientbuf[MAX_IPSTRLEN]; - - if (al->cache.caddr.IsAnyAddr()) { // ICAP OPTIONS xactions lack client - client = "-"; - } else { - if (Config.onoff.log_fqdn) - client = fqdncache_gethostbyaddr(al->cache.caddr, FQDN_LOOKUP_IF_MISS); - if (!client) - client = al->cache.caddr.NtoA(clientbuf, MAX_IPSTRLEN); - } - - user = accessLogFormatName(al->cache.authuser); - - if (!user) - user = accessLogFormatName(al->cache.extuser); - -#if USE_SSL - - if (!user) - user = accessLogFormatName(al->cache.ssluser); +#if USE_FORW_VIA_DB +typedef struct { + hash_link hash; + int n; +} fvdb_entry; +static hash_table *via_table = NULL; +static hash_table *forw_table = NULL; +static void fvdbInit(); +static void fvdbDumpTable(StoreEntry * e, hash_table * hash); +static void fvdbCount(hash_table * hash, const char *key); +static OBJH fvdbDumpVia; +static OBJH fvdbDumpForw; +static FREE fvdbFreeEntry; +static void fvdbClear(void); +static void fvdbRegisterWithCacheManager(); #endif - if (!user) - user = accessLogFormatName(al->cache.rfc931); - - if (user && !*user) - safe_free(user); - - logfilePrintf(logfile, "%9ld.%03d %6d %s -/%03d %"PRId64" %s %s %s -/%s -\n", - (long int) current_time.tv_sec, - (int) current_time.tv_usec / 1000, - - al->icap.trTime, - client, - - al->icap.resStatus, - al->icap.bytesRead, - Adaptation::Icap::ICAP::methodStr(al->icap.reqMethod), - al->icap.reqUri.termedBuf(), - user ? user : dash_str, - al->icap.hostAddr.NtoA(tmp, MAX_IPSTRLEN)); - safe_free(user); -} -#endif +int LogfileStatus = LOG_DISABLE; void accessLogLogTo(customlog* log, AccessLogEntry * al, ACLChecklist * checklist) @@ -2013,32 +119,37 @@ accessLogLogTo(customlog* log, AccessLogEntry * al, ACLChecklist * checklist) switch (log->type) { - case CLF_AUTO: - if (Config.onoff.common_log) - accessLogCommon(al, log->logfile); - else - accessLogSquid(al, log->logfile); + case Log::Format::CLF_SQUID: + Log::Format::SquidNative(al, log->logfile); + break; + + case Log::Format::CLF_COMBINED: + Log::Format::HttpdCombined(al, log->logfile); + break; + + case Log::Format::CLF_COMMON: + Log::Format::HttpdCommon(al, log->logfile); break; - case CLF_SQUID: - accessLogSquid(al, log->logfile); + case Log::Format::CLF_REFERER: + Log::Format::SquidReferer(al, log->logfile); break; - case CLF_COMMON: - accessLogCommon(al, log->logfile); + case Log::Format::CLF_USERAGENT: + Log::Format::SquidUserAgent(al, log->logfile); break; - case CLF_CUSTOM: - accessLogCustom(al, log); + case Log::Format::CLF_CUSTOM: + Log::Format::SquidCustom(al, log); break; #if ICAP_CLIENT - case CLF_ICAP_SQUID: - accessLogICAPSquid(al, log->logfile); + case Log::Format::CLF_ICAP_SQUID: + Log::Format::SquidIcap(al, log->logfile); break; #endif - case CLF_NONE: + case Log::Format::CLF_NONE: return; // abort! default: @@ -2185,8 +296,6 @@ accessLogInit(void) accessLogRegisterWithCacheManager(); - assert(sizeof(log_tags) == (LOG_TYPE_MAX + 1) * sizeof(char *)); - #if USE_ADAPTATION alLogformatHasAdaptToken = false; #endif @@ -2195,7 +304,7 @@ accessLogInit(void) #endif for (log = Config.Log.accesslogs; log; log = log->next) { - if (log->type == CLF_NONE) + if (log->type == Log::Format::CLF_NONE) continue; log->logfile = logfileOpen(log->filename, MAX_URL << 2, 1); @@ -2261,24 +370,6 @@ accessLogInit(void) #endif } -const char * -accessLogTime(time_t t) -{ - - struct tm *tm; - static char buf[128]; - static time_t last_t = 0; - - if (t != last_t) { - tm = localtime(&t); - strftime(buf, 127, "%Y/%m/%d %H:%M:%S", tm); - last_t = t; - } - - return buf; -} - - #if USE_FORW_VIA_DB static void diff --git a/src/main.cc b/src/main.cc index 4ac574d01a..d2fd0c1ec2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -756,8 +756,6 @@ mainReconfigureStart(void) #if ICAP_CLIENT icapLogClose(); #endif - useragentLogClose(); - refererCloseLog(); eventAdd("mainReconfigureFinish", &mainReconfigureFinish, NULL, 0, 1, false); @@ -823,8 +821,6 @@ mainReconfigureFinish(void *) icapLogOpen(); #endif storeLogOpen(); - useragentOpenLog(); - refererOpenLog(); #if USE_DNSSERVERS dnsInit(); @@ -893,15 +889,9 @@ mainRotate(void) storeDirWriteCleanLogs(1); storeLogRotate(); /* store.log */ accessLogRotate(); /* access.log */ - useragentRotateLog(); /* useragent.log */ - refererRotateLog(); /* referer.log */ #if ICAP_CLIENT icapLogRotate(); /*icap.log*/ #endif -#if WIP_FWD_LOG - fwdLogRotate(); -#endif - icmpEngine.Open(); #if USE_DNSSERVERS dnsInit(); @@ -1040,10 +1030,6 @@ mainInitialize(void) externalAclInit(); - useragentOpenLog(); - - refererOpenLog(); - httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ httpReplyInitModule(); /* must go before accepting replies */ @@ -1875,13 +1861,6 @@ SquidShutdown() Store::Root().sync(); /* Flush log writes */ storeLogClose(); accessLogClose(); - useragentLogClose(); - refererCloseLog(); -#if WIP_FWD_LOG - - fwdUninit(); -#endif - Store::Root().sync(); /* Flush log close */ StoreFileSystem::FreeAllFs(); DiskIOModule::FreeAllModules(); diff --git a/src/neighbors.cc b/src/neighbors.cc index 4402bd1077..3490db2329 100644 --- a/src/neighbors.cc +++ b/src/neighbors.cc @@ -1755,7 +1755,7 @@ dump_peers(StoreEntry * sentry, peer * peers) if (e->stats.last_connect_failure) { storeAppendPrintf(sentry, "Last failed connect() at: %s\n", - mkhttpdlogtime(&(e->stats.last_connect_failure))); + Time::FormatHttpd(e->stats.last_connect_failure)); } if (e->peer_domain != NULL) { diff --git a/src/protos.h b/src/protos.h index 12fdcda604..7abeb22046 100644 --- a/src/protos.h +++ b/src/protos.h @@ -57,7 +57,6 @@ SQUIDCEXTERN void fvdbCountForw(const char *key); #if HEADERS_LOG SQUIDCEXTERN void headersLog(int cs, int pq, const HttpRequestMethod& m, void *data); #endif -SQUIDCEXTERN char *log_quote(const char *header); SQUIDCEXTERN int logTypeIsATcpHit(log_type); /* @@ -631,14 +630,6 @@ SQUIDCEXTERN int urlDefaultPort(protocol_t p); SQUIDCEXTERN char *urlHostname(const char *url); SQUIDCEXTERN void urlExtMethodConfigure(void); -SQUIDCEXTERN void useragentOpenLog(void); -SQUIDCEXTERN void useragentRotateLog(void); -SQUIDCEXTERN void logUserAgent(const char *, const char *); -SQUIDCEXTERN void useragentLogClose(void); -SQUIDCEXTERN void refererOpenLog(void); -SQUIDCEXTERN void refererRotateLog(void); -SQUIDCEXTERN void logReferer(const char *, const char *, const char *); -SQUIDCEXTERN void refererCloseLog(void); SQUIDCEXTERN peer_t parseNeighborType(const char *s); /* tools.c */ diff --git a/src/send-announce.cc b/src/send-announce.cc index 04bba820cd..773cceaf86 100644 --- a/src/send-announce.cc +++ b/src/send-announce.cc @@ -92,7 +92,7 @@ send_announce(const ipcache_addrs *ia, const DnsLookupDetails &, void *junk) snprintf(tbuf, 256, "generated %d [%s]\n", (int) squid_curtime, - mkhttpdlogtime(&squid_curtime)); + Time::FormatHttpd(squid_curtime)); strcat(sndbuf, tbuf); l = strlen(sndbuf); diff --git a/src/structs.h b/src/structs.h index 67fd632f61..43490ed6d6 100644 --- a/src/structs.h +++ b/src/structs.h @@ -260,27 +260,10 @@ struct SquidConfig { struct { char *store; char *swap; -#if USE_USERAGENT_LOG - - char *useragent; -#endif -#if USE_REFERER_LOG - - char *referer; -#endif -#if WIP_FWD_LOG - - char *forward; -#endif - - logformat *logformats; - customlog *accesslogs; - #if ICAP_CLIENT customlog *icaplogs; #endif - int rotateNumber; } Log; char *adminEmail; @@ -1314,23 +1297,17 @@ struct _store_rebuild_data { int zero_object_sz; }; -class logformat_token; - -struct _logformat { - char *name; - logformat_token *format; - logformat *next; -}; - class Logfile; +class logformat; +#include "log/Formats.h" struct _customlog { char *filename; ACLList *aclList; logformat *logFormat; Logfile *logfile; customlog *next; - customlog_type type; + Log::Format::log_type type; }; #endif /* SQUID_STRUCTS_H */ diff --git a/src/time.cc b/src/time.cc index f1ef387be4..bb0bbdd9c8 100644 --- a/src/time.cc +++ b/src/time.cc @@ -68,3 +68,66 @@ TimeEngine::tick() { getCurrentTime(); } + +const char * +Time::FormatStrf(time_t t) +{ + struct tm *tm; + static char buf[128]; + static time_t last_t = 0; + + if (t != last_t) { + tm = localtime(&t); + strftime(buf, 127, "%Y/%m/%d %H:%M:%S", tm); + last_t = t; + } + + return buf; +} + +const char * +Time::FormatHttpd(time_t t) +{ + static char buf[128]; + static time_t last_t = 0; + + if (t != last_t) { + struct tm *gmt = gmtime(&t); + +#if !USE_GMT + int gmt_min, gmt_hour, gmt_yday, day_offset; + size_t len; + struct tm *lt; + int min_offset; + + /* localtime & gmtime may use the same static data */ + gmt_min = gmt->tm_min; + gmt_hour = gmt->tm_hour; + gmt_yday = gmt->tm_yday; + + lt = localtime(&t); + + day_offset = lt->tm_yday - gmt_yday; + /* wrap round on end of year */ + if (day_offset > 1) + day_offset = -1; + else if (day_offset < -1) + day_offset = 1; + + min_offset = day_offset * 1440 + (lt->tm_hour - gmt_hour) * 60 + + (lt->tm_min - gmt_min); + + len = strftime(buf, 127 - 5, "%d/%b/%Y:%H:%M:%S ", lt); + snprintf(buf + len, 128 - len, "%+03d%02d", + (min_offset / 60) % 24, + min_offset % 60); +#else /* USE_GMT */ + buf[0] = '\0'; + strftime(buf, 127, "%d/%b/%Y:%H:%M:%S -000", gmt); +#endif /* USE_GMT */ + + last_t = t; + } + + return buf; +} diff --git a/src/typedefs.h b/src/typedefs.h index 9186321b1b..d811b2da22 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -169,8 +169,6 @@ typedef struct _Version Version; typedef struct _link_list link_list; -typedef struct _logformat logformat; - typedef struct _customlog customlog; #if SQUID_SNMP