From 31971e6abe3251b7bb42b7196fdafc82f9c04b7a Mon Sep 17 00:00:00 2001 From: Amos Jeffries Date: Fri, 18 Nov 2011 00:48:25 -0700 Subject: [PATCH] Log Format token namespace upgrade This updates the format parser and storage objects in the Format:: namespace and separates some into separate files. Add a registration API so components can register themselves an array of tokens in a namespace. Registering the arbitrary namespace "example" with some tokens ("a","b") will cause the parser to accept those tokens in a logging format like so: "%example::a %example::b". Future work: - use runners registry instead of Init() function - convert the error pages to use format for the page body macros - convert the %ssl_* tokens in src/ssl/* to use format and "ssl::" - convert external_acl_type to use formats for its helper input string. --- src/client_db.cc | 2 +- src/client_side_reply.cc | 2 +- src/client_side_request.cc | 2 +- src/format/{Tokens.h => ByteCode.h} | 196 ++++++++++++---------------- src/format/Config.cc | 43 ++++++ src/format/Config.h | 76 +++++++++++ src/format/Format.cc | 6 +- src/format/Makefile.am | 8 +- src/format/{Tokens.cc => Token.cc} | 110 +++++++++------- src/format/Token.h | 82 ++++++++++++ src/format/TokenTableEntry.h | 37 ++++++ src/log/FormatHttpdCombined.cc | 2 +- src/log/FormatHttpdCommon.cc | 2 +- src/log/FormatSquidCustom.cc | 1 - src/log/FormatSquidNative.cc | 2 +- src/log/access_log.cc | 6 +- src/main.cc | 3 + src/stat.cc | 2 +- src/store_log.cc | 2 +- 19 files changed, 410 insertions(+), 174 deletions(-) rename src/format/{Tokens.h => ByteCode.h} (69%) create mode 100644 src/format/Config.cc create mode 100644 src/format/Config.h rename src/format/{Tokens.cc => Token.cc} (84%) create mode 100644 src/format/Token.h create mode 100644 src/format/TokenTableEntry.h diff --git a/src/client_db.cc b/src/client_db.cc index dd90241e4b..213ff75579 100644 --- a/src/client_db.cc +++ b/src/client_db.cc @@ -34,7 +34,7 @@ #include "squid.h" #include "event.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "ClientInfo.h" #include "ip/Address.h" #include "mgr/Registration.h" diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc index 0205ffd202..0014fa07cd 100644 --- a/src/client_side_reply.cc +++ b/src/client_side_reply.cc @@ -55,7 +55,7 @@ #endif #include "fde.h" #include "forward.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpReply.h" #include "HttpRequest.h" #include "ip/QosConfig.h" diff --git a/src/client_side_request.cc b/src/client_side_request.cc index 3b0728a7a6..c8f93c45b6 100644 --- a/src/client_side_request.cc +++ b/src/client_side_request.cc @@ -66,7 +66,7 @@ #include "comm/Write.h" #include "compat/inet_pton.h" #include "fde.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpHdrCc.h" #include "HttpReply.h" #include "HttpRequest.h" diff --git a/src/format/Tokens.h b/src/format/ByteCode.h similarity index 69% rename from src/format/Tokens.h rename to src/format/ByteCode.h index 507badffe4..8e345dfcdd 100644 --- a/src/format/Tokens.h +++ b/src/format/ByteCode.h @@ -1,5 +1,5 @@ -#ifndef _SQUID_FMT_TOKENS_H -#define _SQUID_FMT_TOKENS_H +#ifndef _SQUID_FMT_BYTECODE_H +#define _SQUID_FMT_BYTECODE_H /* * Squid configuration allows users to define custom formats in @@ -16,84 +16,49 @@ namespace Format { -#define LOG_BUF_SZ (MAX_URL<<2) - /* * Bytecodes for the configureable format stuff */ typedef enum { LFT_NONE, /* dummy */ + + /* arbitrary string between tokens */ LFT_STRING, + /* client TCP connection remote end details */ LFT_CLIENT_IP_ADDRESS, LFT_CLIENT_FQDN, LFT_CLIENT_PORT, LFT_CLIENT_EUI, - LFT_SERVER_IP_ADDRESS, - LFT_SERVER_FQDN_OR_PEER_NAME, - LFT_SERVER_PORT, - + /* client TCP connection local end details */ LFT_CLIENT_LOCAL_IP, - LFT_LOCAL_LISTENING_IP, LFT_CLIENT_LOCAL_PORT, + /*LFT_CLIENT_LOCAL_FQDN, (rDNS) */ + + /* client connection local squid.conf details */ + LFT_LOCAL_LISTENING_IP, LFT_LOCAL_LISTENING_PORT, - /*LFT_LOCAL_NAME, */ + /*LFT_LOCAL_LISTENING_NAME, (myportname) */ + + /* server TCP connection remote end details */ + LFT_SERVER_IP_ADDRESS, + LFT_SERVER_FQDN_OR_PEER_NAME, + LFT_SERVER_PORT, + /* server TCP connection local end details */ LFT_SERVER_LOCAL_IP, LFT_SERVER_LOCAL_IP_OLD_27, LFT_SERVER_LOCAL_PORT, - LFT_TIME_SECONDS_SINCE_EPOCH, - LFT_TIME_SUBSECOND, - LFT_TIME_LOCALTIME, - LFT_TIME_GMT, - LFT_TIME_TO_HANDLE_REQUEST, - - LFT_PEER_RESPONSE_TIME, - LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, - LFT_DNS_WAIT_TIME, - - LFT_REQUEST_HEADER, - LFT_REQUEST_HEADER_ELEM, - LFT_REQUEST_ALL_HEADERS, - - LFT_ADAPTED_REQUEST_HEADER, - LFT_ADAPTED_REQUEST_HEADER_ELEM, - LFT_ADAPTED_REQUEST_ALL_HEADERS, - - LFT_REPLY_HEADER, - LFT_REPLY_HEADER_ELEM, - LFT_REPLY_ALL_HEADERS, - - LFT_USER_NAME, - LFT_USER_LOGIN, - LFT_USER_IDENT, - /*LFT_USER_REALM, */ - /*LFT_USER_SCHEME, */ - LFT_USER_EXTERNAL, - - LFT_HTTP_SENT_STATUS_CODE_OLD_30, - LFT_HTTP_SENT_STATUS_CODE, - LFT_HTTP_RECEIVED_STATUS_CODE, - /*LFT_HTTP_STATUS, */ - LFT_HTTP_BODY_BYTES_READ, - - LFT_SQUID_STATUS, - LFT_SQUID_ERROR, - LFT_SQUID_ERROR_DETAIL, - LFT_SQUID_HIERARCHY, - - LFT_MIME_TYPE, - - /* original Request-Line details receved from client */ + /* original Request-Line details recieved from client */ LFT_CLIENT_REQ_METHOD, LFT_CLIENT_REQ_URI, LFT_CLIENT_REQ_URLPATH, /* LFT_CLIENT_REQ_QUERY, */ LFT_CLIENT_REQ_VERSION, - /* Request-Line details receved from client (legacy, filtered) */ + /* Request-Line details recieved from client (legacy, filtered) */ LFT_REQUEST_METHOD, LFT_REQUEST_URI, LFT_REQUEST_URLPATH_OLD_31, @@ -101,6 +66,16 @@ typedef enum { LFT_REQUEST_VERSION_OLD_2X, LFT_REQUEST_VERSION, + /* request header details pre-adaptation */ + LFT_REQUEST_HEADER, + LFT_REQUEST_HEADER_ELEM, + LFT_REQUEST_ALL_HEADERS, + + /* request header details post-adaptation */ + LFT_ADAPTED_REQUEST_HEADER, + LFT_ADAPTED_REQUEST_HEADER_ELEM, + LFT_ADAPTED_REQUEST_ALL_HEADERS, + /* Request-Line details sent to the server/peer */ LFT_SERVER_REQ_METHOD, LFT_SERVER_REQ_URI, @@ -108,12 +83,37 @@ typedef enum { /*LFT_SERVER_REQ_QUERY, */ LFT_SERVER_REQ_VERSION, + /* request meta details */ LFT_REQUEST_SIZE_TOTAL, /*LFT_REQUEST_SIZE_LINE, */ LFT_REQUEST_SIZE_HEADERS, /*LFT_REQUEST_SIZE_BODY, */ /*LFT_REQUEST_SIZE_BODY_NO_TE, */ + /* original Status-Line details recieved from server */ + // XXX: todo + + /* Status-Line details sent to the client */ + // XXX: todo + + /* response Status-Line details (legacy, filtered) */ + LFT_HTTP_SENT_STATUS_CODE_OLD_30, + LFT_HTTP_SENT_STATUS_CODE, + LFT_HTTP_RECEIVED_STATUS_CODE, + /*LFT_HTTP_STATUS, */ + LFT_HTTP_BODY_BYTES_READ, + + /* response header details pre-adaptation */ + LFT_REPLY_HEADER, + LFT_REPLY_HEADER_ELEM, + LFT_REPLY_ALL_HEADERS, + + /* response header details post-adaptation */ + /* LFT_ADAPTED_REPLY_HEADER, */ + /* LFT_ADAPTED_REPLY_HEADER_ELEM, */ + /* LFT_ADAPTED_REPLY_ALL_HEADERS, */ + + /* response meta details */ LFT_REPLY_SIZE_TOTAL, LFT_REPLY_HIGHOFFSET, LFT_REPLY_OBJECTSIZE, @@ -122,6 +122,34 @@ typedef enum { /*LFT_REPLY_SIZE_BODY, */ /*LFT_REPLY_SIZE_BODY_NO_TE, */ + /* client credentials */ + LFT_USER_NAME, /* any source will do */ + LFT_USER_LOGIN, + LFT_USER_IDENT, + /*LFT_USER_REALM, */ + /*LFT_USER_SCHEME, */ + LFT_USER_EXTERNAL, + /* LFT_USER_SSL_CERT, */ + + /* global time details */ + LFT_TIME_SECONDS_SINCE_EPOCH, + LFT_TIME_SUBSECOND, + LFT_TIME_LOCALTIME, + LFT_TIME_GMT, + + /* processing time details */ + LFT_TIME_TO_HANDLE_REQUEST, + LFT_PEER_RESPONSE_TIME, + LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, + LFT_DNS_WAIT_TIME, + + /* Squid internal processing details */ + LFT_SQUID_STATUS, + LFT_SQUID_ERROR, + LFT_SQUID_ERROR_DETAIL, + LFT_SQUID_HIERARCHY, + + LFT_MIME_TYPE, LFT_TAG, LFT_IO_SIZE_TOTAL, LFT_EXT_LOG, @@ -129,8 +157,8 @@ typedef enum { LFT_SEQUENCE_NUMBER, #if USE_ADAPTATION - LTF_ADAPTATION_SUM_XACT_TIMES, - LTF_ADAPTATION_ALL_XACT_TIMES, + LFT_ADAPTATION_SUM_XACT_TIMES, + LFT_ADAPTATION_ALL_XACT_TIMES, LFT_ADAPTATION_LAST_HEADER, LFT_ADAPTATION_LAST_HEADER_ELEM, LFT_ADAPTATION_LAST_ALL_HEADERS, @@ -174,62 +202,8 @@ enum Quoting { LOG_QUOTE_RAW }; -struct TokenTableEntry { - const char *config; - ByteCode_t token_type; - int options; -}; - -// XXX: inherit from linked list -class Token -{ -public: - Token() : type(LFT_NONE), - label(NULL), - widthMin(-1), - widthMax(-1), - quote(LOG_QUOTE_NONE), - left(0), - space(0), - zero(0), - divisor(0), - next(NULL) - { data.string = NULL; }; - ~Token(); - - /** parses a single token. Returns the token length in characters, - * and fills in this item with the token information. - * def is for sure null-terminated. - */ - int parse(char *def, enum Quoting *quote); - - ByteCode_t type; - const char *label; - union { - char *string; - - struct { - char *header; - char *element; - char separator; - } header; - char *timespec; - } data; - int widthMin; ///< minimum field width - int widthMax; ///< maximum field width - enum Quoting quote; - unsigned int left:1; - unsigned int space:1; - unsigned int zero:1; - int divisor; - Token *next; /* todo: move from linked list to array */ - -private: - char *scanForToken(const struct TokenTableEntry *table, char *cur); -}; - extern const char *log_tags[]; } // namespace Format -#endif /* _SQUID_FMT_TOKENS_H */ +#endif /* _SQUID_FMT_BYTECODE_H */ diff --git a/src/format/Config.cc b/src/format/Config.cc new file mode 100644 index 0000000000..7ac1272d2f --- /dev/null +++ b/src/format/Config.cc @@ -0,0 +1,43 @@ +#include "config.h" +#include "format/Config.h" +#include "protos.h" +#include + +Format::FmtConfig Format::TheConfig; + +void +Format::FmtConfig::parseFormats() +{ + char *name, *def; + + if ((name = strtok(NULL, w_space)) == NULL) + self_destruct(); + + if ((def = strtok(NULL, "\r\n")) == NULL) { + self_destruct(); + return; + } + + debugs(3, 2, "Custom Format for '" << name << "' is '" << def << "'"); + + Format *nlf = new Format(name); + + if (!nlf->parse(def)) { + self_destruct(); + return; + } + + // add to global config list + nlf->next = formats; + formats = nlf; +} + +void +Format::FmtConfig::registerTokens(const String &nsName, TokenTableEntry const *tokenArray) +{ + debugs(46, 2, HERE << " register format tokens for '" << nsName << "'"); + if (tokenArray != NULL) + tokens.push_back(TokenNamespace(nsName, tokenArray)); + else + debugs(0,0, "BUG: format tokens for '" << nsName << "' missing!"); +} diff --git a/src/format/Config.h b/src/format/Config.h new file mode 100644 index 0000000000..e4c4882ee5 --- /dev/null +++ b/src/format/Config.h @@ -0,0 +1,76 @@ +#ifndef SQUID_SRC_FORMAT_CONFIG_H +#define SQUID_SRC_FORMAT_CONFIG_H + +#include "format/Format.h" +//#include "format/TokenTableEntry.h" +#include "SquidString.h" +#include + +class StoreEntry; + +namespace Format +{ + +class TokenTableEntry; + +/// A namespace or 'set' of tokens +/// components register their namespace prefix and an array of tokens +/// which can then be embeded in any format. +class TokenNamespace +{ +public: + TokenNamespace(const String &nsName, TokenTableEntry const *tSet) : prefix(nsName), tokenSet(tSet) {} + + /// prefix namespace name (excluding '::') + String prefix; + + /// array of tokens inside this namespace + /// The set of tokens may change, but the location of it pointed to from here must not. + TokenTableEntry const *tokenSet; +}; + +/// The set of custom formats defined in squid.conf +/// +class FmtConfig +{ +public: + /// Parse a log format directive line (logfile_format) + void parseFormats(); + + /// Dump/display the formats currently known to the provided StoreEntry object + void dumpFormats(StoreEntry *e, const char *name) { + formats->dump(e, name); + } + + /* Register a namespace set of tokens to be accepted by the format parser. + * Multiple arrays can be registered, they will be scanned for + * in order registered. So care needs to be taken that arrays registered + * first do not overlap or consume tokens registered later for a namespace. + */ + void registerTokens(const String &nsName, TokenTableEntry const *tokenArray); + + /// Linked list of custom formats + Format *formats; + + /// list of token namespaces registered + std::list tokens; + +#if USE_ADAPTATION + bool hasAdaptToken; +#endif + +#if ICAP_CLIENT + bool hasIcapToken; +#endif +}; + +extern FmtConfig TheConfig; + +} // namespace Format + +// Legacy parsing wrappers +#define parse_format(X) (X)->parseFormats() +#define free_format(X) do{ delete (*(X)).formats; (*(X)).formats=NULL; }while(false) +#define dump_format(E,N,D) (D).dumpFormats((E),(N)) + +#endif diff --git a/src/format/Format.cc b/src/format/Format.cc index 6fb53f6463..13708abed8 100644 --- a/src/format/Format.cc +++ b/src/format/Format.cc @@ -5,7 +5,7 @@ #include "errorpage.h" #include "format/Format.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpRequest.h" #include "MemBuf.h" #include "rfc1738.h" @@ -516,7 +516,7 @@ Format::Format::assemble(MemBuf &mb, AccessLogEntry *al, int logSequenceNumber) break; #if USE_ADAPTATION - case LTF_ADAPTATION_SUM_XACT_TIMES: + case LFT_ADAPTATION_SUM_XACT_TIMES: if (al->request) { Adaptation::History::Pointer ah = al->request->adaptHistory(); if (ah != NULL) @@ -525,7 +525,7 @@ Format::Format::assemble(MemBuf &mb, AccessLogEntry *al, int logSequenceNumber) } break; - case LTF_ADAPTATION_ALL_XACT_TIMES: + case LFT_ADAPTATION_ALL_XACT_TIMES: if (al->request) { Adaptation::History::Pointer ah = al->request->adaptHistory(); if (ah != NULL) diff --git a/src/format/Makefile.am b/src/format/Makefile.am index cbf05deb11..d8e1c24a9e 100644 --- a/src/format/Makefile.am +++ b/src/format/Makefile.am @@ -4,10 +4,14 @@ include $(top_srcdir)/src/TestHeaders.am noinst_LTLIBRARIES = libformat.la libformat_la_SOURCES = \ + ByteCode.h \ + Config.cc \ + Config.h \ Format.cc \ Format.h \ Quoting.cc \ Quoting.h \ - Tokens.cc \ - Tokens.h + Token.cc \ + Token.h \ + TokenTableEntry.h diff --git a/src/format/Tokens.cc b/src/format/Token.cc similarity index 84% rename from src/format/Tokens.cc rename to src/format/Token.cc index f27203e9e5..5a61d8330e 100644 --- a/src/format/Tokens.cc +++ b/src/format/Token.cc @@ -1,5 +1,7 @@ #include "config.h" -#include "format/Tokens.h" +#include "format/Config.h" +#include "format/Token.h" +#include "format/TokenTableEntry.h" #include "Store.h" const char *Format::log_tags[] = { @@ -36,7 +38,7 @@ namespace Format { /// 1-char tokens. -static struct TokenTableEntry TokenTable1C[] = { +static TokenTableEntry TokenTable1C[] = { {">a", LFT_CLIENT_IP_ADDRESS}, {">p", LFT_CLIENT_PORT}, @@ -59,7 +61,7 @@ static struct TokenTableEntry TokenTable1C[] = { }; /// 2-char tokens -static struct TokenTableEntry TokenTable2C[] = { +static TokenTableEntry TokenTable2C[] = { {">la", LFT_CLIENT_LOCAL_IP}, {"la", LFT_LOCAL_LISTENING_IP}, @@ -143,19 +145,26 @@ static struct TokenTableEntry TokenTable2C[] = { {NULL, LFT_NONE} /* this must be last */ }; +/// Miscellaneous >2 byte tokens +static TokenTableEntry TokenTableMisc[] = { + {">eui", LFT_CLIENT_EUI}, + {"err_code", LFT_SQUID_ERROR }, + {"err_detail", LFT_SQUID_ERROR_DETAIL }, + {NULL, LFT_NONE} /* this must be last */ +}; + #if USE_ADAPTATION -/// Adaptation (adapt::) tokens -static struct TokenTableEntry TokenTableAdapt[] = { - {"all_trs", LTF_ADAPTATION_ALL_XACT_TIMES}, - {"sum_trs", LTF_ADAPTATION_SUM_XACT_TIMES}, +static TokenTableEntry TokenTableAdapt[] = { + {"all_trs", LFT_ADAPTATION_ALL_XACT_TIMES}, + {"sum_trs", LFT_ADAPTATION_SUM_XACT_TIMES}, {"st", LFT_ICAP_BYTES_SENT}, - {"st", LFT_ICAP_BYTES_SENT}, + {"h", LFT_ICAP_REQ_HEADER}, {"2 byte tokens -static struct TokenTableEntry TokenTableMisc[] = { - {">eui", LFT_CLIENT_EUI}, - {"err_code", LFT_SQUID_ERROR }, - {"err_detail", LFT_SQUID_ERROR_DETAIL }, - {NULL, LFT_NONE} /* this must be last */ -}; - } // namespace Format +/// Register all components custom format tokens +void +Format::Token::Init() +{ + // TODO standard log tokens + // TODO external ACL fmt tokens + +#if USE_ADAPTATION + TheConfig.registerTokens(String("adapt"),::Format::TokenTableAdapt); +#endif +#if ICAP_CLIENT + TheConfig.registerTokens(String("icap"),::Format::TokenTableIcap); +#endif + + // TODO tokens for OpenSSL errors in "ssl::" +} + /// Scans a token table to see if the next token exists there /// returns a pointer to next unparsed byte and updates type member if found char * -Format::Token::scanForToken(const struct TokenTableEntry *table, char *cur) +Format::Token::scanForToken(TokenTableEntry const table[], char *cur) { - for (const struct TokenTableEntry *lte = table; lte->config != NULL; lte++) { - if (strncmp(lte->config, cur, strlen(lte->config)) == 0) { - type = lte->token_type; - label = lte->config; + for (TokenTableEntry const *lte = table; lte->configTag != NULL; lte++) { + debugs(46, 8, HERE << "compare tokens '" << lte->configTag << "' with '" << cur << "'"); + if (strncmp(lte->configTag, cur, strlen(lte->configTag)) == 0) { + type = lte->tokenType; + label = lte->configTag; debugs(46, 7, HERE << "Found token '" << label << "'"); - return cur + strlen(lte->config); + return cur + strlen(lte->configTag); } } return cur; @@ -322,26 +341,25 @@ Format::Token::parse(char *def, Quoting *quoting) type = LFT_NONE; - // Scan each token namespace - if (strncmp(cur, "icap::", 6) == 0) { -#if ICAP_CLIENT - cur += 6; - debugs(46, 5, HERE << "scan for icap:: token"); - cur = scanForToken(TokenTableIcap, cur); -#else - debugs(46, DBG_IMPORTANT, "ERROR: Format uses icap:: token. ICAP disabled!"); -#endif - } else if (strncmp(cur, "adapt::", 7) == 0) { -#if USE_ADAPTATION - cur += 7; - debugs(46, 5, HERE << "scan for adapt:: token"); - cur = scanForToken(TokenTableAdapt, cur); -#else - debugs(46, DBG_IMPORTANT, "ERROR: Format uses adapt:: token. Adaptation disabled!"); -#endif - } else { + // Scan each registered token namespace + debugs(46, 9, HERE << "check for token in " << TheConfig.tokens.size() << " namespaces."); + for (std::list::const_iterator itr = TheConfig.tokens.begin(); itr != TheConfig.tokens.end(); itr++) { + debugs(46, 7, HERE << "check for possible " << itr->prefix << ":: token"); + const size_t len = itr->prefix.size(); + if (itr->prefix.cmp(cur, len) == 0 && cur[len] == ':' && cur[len+1] == ':') { + debugs(46, 5, HERE << "check for " << itr->prefix << ":: token in '" << cur << "'"); + const char *old = cur; + cur = scanForToken(itr->tokenSet, cur+len+2); + if (old != cur) // found + break; + else // reset to start of namespace + cur = cur - len - 2; + } + } + + if (type == LFT_NONE) { // For upward compatibility, assume "http::" prefix as default prefix - // for all log access formating codes, except those starting with a + // for all log access formatting codes, except those starting with a // "%" or a known namespace. (ie "icap::", "adapt::") if (strncmp(cur,"http::", 6) == 0 && *(cur+6) != '%' ) cur += 6; diff --git a/src/format/Token.h b/src/format/Token.h new file mode 100644 index 0000000000..8951e40b6b --- /dev/null +++ b/src/format/Token.h @@ -0,0 +1,82 @@ +#ifndef _SQUID_FORMAT_TOKEN_H +#define _SQUID_FORMAT_TOKEN_H + +//#include "format/TokenTableEntry.h" +#include "format/ByteCode.h" + +/* + * Squid configuration allows users to define custom formats in + * several components. + * - logging + * - external ACL input + * - deny page URL + * + * These enumerations and classes define the API for parsing of + * format directives to define these patterns. Along with output + * functionality to produce formatted buffers. + */ + +namespace Format +{ + +class TokenTableEntry; + +#define LOG_BUF_SZ (MAX_URL<<2) + +// XXX: inherit from linked list +class Token +{ +public: + Token() : type(LFT_NONE), + label(NULL), + widthMin(-1), + widthMax(-1), + quote(LOG_QUOTE_NONE), + left(0), + space(0), + zero(0), + divisor(0), + next(NULL) + { data.string = NULL; } + + ~Token(); + + /// Initialize the format token registrations + static void Init(); + + /** parses a single token. Returns the token length in characters, + * and fills in this item with the token information. + * def is for sure null-terminated. + */ + int parse(char *def, enum Quoting *quote); + + ByteCode_t type; + const char *label; + union { + char *string; + + struct { + char *header; + char *element; + char separator; + } header; + char *timespec; + } data; + int widthMin; ///< minimum field width + int widthMax; ///< maximum field width + enum Quoting quote; + unsigned int left:1; + unsigned int space:1; + unsigned int zero:1; + int divisor; + Token *next; /* todo: move from linked list to array */ + +private: + char *scanForToken(TokenTableEntry const table[], char *cur); +}; + +extern const char *log_tags[]; + +} // namespace Format + +#endif /* _SQUID_FORMAT_TOKEN_H */ diff --git a/src/format/TokenTableEntry.h b/src/format/TokenTableEntry.h new file mode 100644 index 0000000000..a1fa4dfa81 --- /dev/null +++ b/src/format/TokenTableEntry.h @@ -0,0 +1,37 @@ +#ifndef _SQUID_FORMAT_TOKENTABLEENTRY_H +#define _SQUID_FORMAT_TOKENTABLEENTRY_H + +#include "format/ByteCode.h" + +/* + * Squid configuration allows users to define custom formats in + * several components. + * - logging + * - external ACL input + * - deny page URL + * + * These enumerations and classes define the API for parsing of + * format directives to define these patterns. Along with output + * functionality to produce formatted buffers. + */ + +namespace Format +{ + +/// One entry in a table of format tokens. +class TokenTableEntry { +public: + /// the config file ASCII representation for this token + /// just the base tag bytes, excluding any option syntax bytes + const char *configTag; + + /// the internal byte code representatio of this token + ByteCode_t tokenType; + + /// 32-bit mask? of options affecting the output display of this token + uint32_t options; +}; + +} // namespace Format + +#endif /* _SQUID_FORMAT_TOKENTABLEENTRY_H */ diff --git a/src/log/FormatHttpdCombined.cc b/src/log/FormatHttpdCombined.cc index 3283956126..72bc6daba0 100644 --- a/src/log/FormatHttpdCombined.cc +++ b/src/log/FormatHttpdCombined.cc @@ -34,7 +34,7 @@ #include "config.h" #include "AccessLogEntry.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "format/Quoting.h" #include "HttpRequest.h" #include "log/File.h" diff --git a/src/log/FormatHttpdCommon.cc b/src/log/FormatHttpdCommon.cc index a44d282f15..3dce6a5ae8 100644 --- a/src/log/FormatHttpdCommon.cc +++ b/src/log/FormatHttpdCommon.cc @@ -35,7 +35,7 @@ #include "config.h" #include "AccessLogEntry.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "log/File.h" #include "log/Formats.h" #include "SquidTime.h" diff --git a/src/log/FormatSquidCustom.cc b/src/log/FormatSquidCustom.cc index bf43fe0720..1a94b5170b 100644 --- a/src/log/FormatSquidCustom.cc +++ b/src/log/FormatSquidCustom.cc @@ -34,7 +34,6 @@ #include "config.h" #include "AccessLogEntry.h" -#include "format/Tokens.h" #include "log/File.h" #include "log/Formats.h" #include "MemBuf.h" diff --git a/src/log/FormatSquidNative.cc b/src/log/FormatSquidNative.cc index 55ac703cd5..76d3107194 100644 --- a/src/log/FormatSquidNative.cc +++ b/src/log/FormatSquidNative.cc @@ -35,7 +35,7 @@ #include "config.h" #include "AccessLogEntry.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "log/File.h" #include "log/Formats.h" #include "SquidTime.h" diff --git a/src/log/access_log.cc b/src/log/access_log.cc index 228fee6f3d..f36e865282 100644 --- a/src/log/access_log.cc +++ b/src/log/access_log.cc @@ -47,7 +47,7 @@ #include "eui/Eui48.h" #include "eui/Eui64.h" #endif -#include "format/Tokens.h" +#include "format/Token.h" #include "hier_code.h" #include "HttpReply.h" #include "HttpRequest.h" @@ -322,8 +322,8 @@ accessLogInit(void) #if USE_ADAPTATION for (Format::Token * curr_token = (log->logFormat?log->logFormat->format:NULL); curr_token; curr_token = curr_token->next) { - if (curr_token->type == Format::LTF_ADAPTATION_SUM_XACT_TIMES || - curr_token->type == Format::LTF_ADAPTATION_ALL_XACT_TIMES || + if (curr_token->type == Format::LFT_ADAPTATION_SUM_XACT_TIMES || + curr_token->type == Format::LFT_ADAPTATION_ALL_XACT_TIMES || curr_token->type == Format::LFT_ADAPTATION_LAST_HEADER || curr_token->type == Format::LFT_ADAPTATION_LAST_HEADER_ELEM || curr_token->type == Format::LFT_ADAPTATION_LAST_ALL_HEADERS) { diff --git a/src/main.cc b/src/main.cc index 6d17cb60b3..5fdc1c9b89 100644 --- a/src/main.cc +++ b/src/main.cc @@ -63,6 +63,7 @@ #include "event.h" #include "EventLoop.h" #include "ExternalACL.h" +#include "format/Token.h" #include "fs/Module.h" #include "PeerSelectState.h" #include "Store.h" @@ -1391,6 +1392,8 @@ SquidMain(int argc, char **argv) #endif Ip::ProbeTransport(); // determine IPv4 or IPv6 capabilities before parsing. + Format::Token::Init(); // XXX: temporary. Use a runners registry of pre-parse runners instead. + parse_err = parseConfigFile(ConfigFile); Mem::Report(); diff --git a/src/stat.cc b/src/stat.cc index 70fd04b198..be507e01b1 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -34,7 +34,7 @@ #include "squid.h" #include "event.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "StoreClient.h" #if USE_AUTH #include "auth/UserRequest.h" diff --git a/src/store_log.cc b/src/store_log.cc index c68c377162..ddbce3be7e 100644 --- a/src/store_log.cc +++ b/src/store_log.cc @@ -33,7 +33,7 @@ */ #include "squid.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpReply.h" #include "log/File.h" #include "MemObject.h" -- 2.39.5