2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
10 #include "format/Config.h"
11 #include "format/Token.h"
12 #include "format/TokenTableEntry.h"
14 #include "SquidConfig.h"
17 // Due to token overlaps between 1 and 2 letter tokens (Bug 3310)
18 // We split the token table into sets determined by the token length
23 static TokenTableEntry TokenTable1C
[] = {
25 TokenTableEntry(">a", LFT_CLIENT_IP_ADDRESS
),
26 TokenTableEntry(">p", LFT_CLIENT_PORT
),
27 TokenTableEntry(">A", LFT_CLIENT_FQDN
),
29 TokenTableEntry("<a", LFT_SERVER_IP_ADDRESS
),
30 TokenTableEntry("<p", LFT_SERVER_PORT
),
31 TokenTableEntry("<A", LFT_SERVER_FQDN_OR_PEER_NAME
),
33 TokenTableEntry(">h", LFT_REQUEST_HEADER
),
34 TokenTableEntry(">h", LFT_REQUEST_ALL_HEADERS
),
35 TokenTableEntry("<h", LFT_REPLY_HEADER
),
36 TokenTableEntry("<h", LFT_REPLY_ALL_HEADERS
),
38 TokenTableEntry(">v", LFT_REQUEST_VERSION_OLD_2X
),
40 TokenTableEntry("%", LFT_PERCENT
),
42 TokenTableEntry(NULL
, LFT_NONE
) /* this must be last */
46 static TokenTableEntry TokenTable2C
[] = {
48 TokenTableEntry(">la", LFT_CLIENT_LOCAL_IP
),
49 TokenTableEntry("la", LFT_LOCAL_LISTENING_IP
),
50 TokenTableEntry(">lp", LFT_CLIENT_LOCAL_PORT
),
51 TokenTableEntry("lp", LFT_LOCAL_LISTENING_PORT
),
52 /*TokenTableEntry( "lA", LFT_LOCAL_NAME ), */
54 TokenTableEntry("<la", LFT_SERVER_LOCAL_IP
),
55 TokenTableEntry("oa", LFT_SERVER_LOCAL_IP_OLD_27
),
56 TokenTableEntry("<lp", LFT_SERVER_LOCAL_PORT
),
58 TokenTableEntry("ts", LFT_TIME_SECONDS_SINCE_EPOCH
),
59 TokenTableEntry("tu", LFT_TIME_SUBSECOND
),
60 TokenTableEntry("tl", LFT_TIME_LOCALTIME
),
61 TokenTableEntry("tg", LFT_TIME_GMT
),
62 TokenTableEntry("tS", LFT_TIME_START
),
63 TokenTableEntry("tr", LFT_TIME_TO_HANDLE_REQUEST
),
65 TokenTableEntry("<pt", LFT_PEER_RESPONSE_TIME
),
66 TokenTableEntry("<tt", LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME
),
67 TokenTableEntry("dt", LFT_DNS_WAIT_TIME
),
69 TokenTableEntry(">ha", LFT_ADAPTED_REQUEST_HEADER
),
70 TokenTableEntry(">ha", LFT_ADAPTED_REQUEST_ALL_HEADERS
),
72 TokenTableEntry("un", LFT_USER_NAME
),
73 TokenTableEntry("ul", LFT_USER_LOGIN
),
74 /*TokenTableEntry( "ur", LFT_USER_REALM ), */
75 /*TokenTableEntry( "us", LFT_USER_SCHEME ), */
76 TokenTableEntry("ui", LFT_USER_IDENT
),
77 TokenTableEntry("ue", LFT_USER_EXTERNAL
),
79 TokenTableEntry("Hs", LFT_HTTP_SENT_STATUS_CODE_OLD_30
),
80 TokenTableEntry(">Hs", LFT_HTTP_SENT_STATUS_CODE
),
81 TokenTableEntry("<Hs", LFT_HTTP_RECEIVED_STATUS_CODE
),
82 /*TokenTableEntry( "Ht", LFT_HTTP_STATUS ), */
83 TokenTableEntry("<bs", LFT_HTTP_BODY_BYTES_READ
),
85 TokenTableEntry("Ss", LFT_SQUID_STATUS
),
86 TokenTableEntry("Sh", LFT_SQUID_HIERARCHY
),
88 TokenTableEntry("mt", LFT_MIME_TYPE
),
90 TokenTableEntry(">rm", LFT_CLIENT_REQ_METHOD
),
91 TokenTableEntry(">ru", LFT_CLIENT_REQ_URI
),
92 TokenTableEntry(">rs", LFT_CLIENT_REQ_URLSCHEME
),
93 TokenTableEntry(">rd", LFT_CLIENT_REQ_URLDOMAIN
),
94 TokenTableEntry(">rP", LFT_CLIENT_REQ_URLPORT
),
95 TokenTableEntry(">rp", LFT_CLIENT_REQ_URLPATH
),
96 /*TokenTableEntry(">rq", LFT_CLIENT_REQ_QUERY),*/
97 TokenTableEntry(">rv", LFT_CLIENT_REQ_VERSION
),
99 TokenTableEntry("rm", LFT_REQUEST_METHOD
),
100 TokenTableEntry("ru", LFT_REQUEST_URI
), /* doesn't include the query-string */
101 TokenTableEntry("rp", LFT_REQUEST_URLPATH_OLD_31
),
102 /* TokenTableEntry( "rq", LFT_REQUEST_QUERY ), * / / * the query-string, INCLUDING the leading ? */
103 TokenTableEntry("rv", LFT_REQUEST_VERSION
),
104 TokenTableEntry("rG", LFT_REQUEST_URLGROUP_OLD_2X
),
106 TokenTableEntry("<rm", LFT_SERVER_REQ_METHOD
),
107 TokenTableEntry("<ru", LFT_SERVER_REQ_URI
),
108 TokenTableEntry("<rs", LFT_SERVER_REQ_URLSCHEME
),
109 TokenTableEntry("<rd", LFT_SERVER_REQ_URLDOMAIN
),
110 TokenTableEntry("<rP", LFT_SERVER_REQ_URLPORT
),
111 TokenTableEntry("<rp", LFT_SERVER_REQ_URLPATH
),
112 /*TokenTableEntry("<rq", LFT_SERVER_REQ_QUERY),*/
113 TokenTableEntry("<rv", LFT_SERVER_REQ_VERSION
),
115 TokenTableEntry(">st", LFT_CLIENT_REQUEST_SIZE_TOTAL
),
116 TokenTableEntry(">sh", LFT_CLIENT_REQUEST_SIZE_HEADERS
),
117 /*TokenTableEntry( ">sb", LFT_REQUEST_SIZE_BODY ), */
118 /*TokenTableEntry( ">sB", LFT_REQUEST_SIZE_BODY_NO_TE ), */
120 TokenTableEntry("<st", LFT_ADAPTED_REPLY_SIZE_TOTAL
), // XXX: adapted should be code: <sta
121 TokenTableEntry("<sH", LFT_REPLY_HIGHOFFSET
),
122 TokenTableEntry("<sS", LFT_REPLY_OBJECTSIZE
),
123 TokenTableEntry("<sh", LFT_ADAPTED_REPLY_SIZE_HEADERS
), // XXX: adapted should be code: <sha
124 /*TokenTableEntry( "<sb", LFT_REPLY_SIZE_BODY ), */
125 /*TokenTableEntry( "<sB", LFT_REPLY_SIZE_BODY_NO_TE ), */
127 TokenTableEntry("st", LFT_CLIENT_IO_SIZE_TOTAL
), // XXX: total from client should be stC ??
128 /*TokenTableEntry("stP", LFT_SERVER_IO_SIZE_TOTAL),*/
130 TokenTableEntry("et", LFT_TAG
),
131 TokenTableEntry("ea", LFT_EXT_LOG
),
132 TokenTableEntry("sn", LFT_SEQUENCE_NUMBER
),
134 TokenTableEntry(NULL
, LFT_NONE
) /* this must be last */
137 /// Miscellaneous >2 byte tokens
138 static TokenTableEntry TokenTableMisc
[] = {
139 TokenTableEntry(">eui", LFT_CLIENT_EUI
),
140 TokenTableEntry(">qos", LFT_CLIENT_LOCAL_TOS
),
141 TokenTableEntry("<qos", LFT_SERVER_LOCAL_TOS
),
142 TokenTableEntry(">nfmark", LFT_CLIENT_LOCAL_NFMARK
),
143 TokenTableEntry("<nfmark", LFT_SERVER_LOCAL_NFMARK
),
144 TokenTableEntry("err_code", LFT_SQUID_ERROR
),
145 TokenTableEntry("err_detail", LFT_SQUID_ERROR_DETAIL
),
146 TokenTableEntry("note", LFT_NOTE
),
147 TokenTableEntry("credentials", LFT_CREDENTIALS
),
149 * Legacy external_acl_type format tokens
151 TokenTableEntry("ACL", LFT_EXT_ACL_NAME
),
152 TokenTableEntry("DATA", LFT_EXT_ACL_DATA
),
153 TokenTableEntry("DST", LFT_CLIENT_REQ_URLDOMAIN
),
154 TokenTableEntry("EXT_LOG", LFT_EXT_LOG
),
155 TokenTableEntry("EXT_USER", LFT_USER_EXTERNAL
),
156 TokenTableEntry("IDENT", LFT_USER_IDENT
),
157 TokenTableEntry("LOGIN", LFT_USER_LOGIN
),
158 TokenTableEntry("METHOD", LFT_CLIENT_REQ_METHOD
),
159 TokenTableEntry("MYADDR", LFT_LOCAL_LISTENING_IP
),
160 TokenTableEntry("MYPORT", LFT_LOCAL_LISTENING_PORT
),
161 TokenTableEntry("PATH", LFT_CLIENT_REQ_URLPATH
),
162 TokenTableEntry("PORT", LFT_CLIENT_REQ_URLPORT
),
163 TokenTableEntry("PROTO", LFT_CLIENT_REQ_URLSCHEME
),
164 TokenTableEntry("SRCEUI48", LFT_EXT_ACL_CLIENT_EUI48
),
165 TokenTableEntry("SRCEUI64", LFT_EXT_ACL_CLIENT_EUI64
),
166 TokenTableEntry("SRCPORT", LFT_CLIENT_PORT
),
167 TokenTableEntry("SRC", LFT_CLIENT_IP_ADDRESS
), // keep after longer SRC* tokens
168 TokenTableEntry("TAG", LFT_TAG
),
169 TokenTableEntry("URI", LFT_CLIENT_REQ_URI
),
171 TokenTableEntry("USER_CERTCHAIN", LFT_EXT_ACL_USER_CERTCHAIN_RAW
),
172 TokenTableEntry("USER_CERT", LFT_EXT_ACL_USER_CERT_RAW
),
174 TokenTableEntry(NULL
, LFT_NONE
) /* this must be last */
178 static TokenTableEntry TokenTableAdapt
[] = {
179 TokenTableEntry("all_trs", LFT_ADAPTATION_ALL_XACT_TIMES
),
180 TokenTableEntry("sum_trs", LFT_ADAPTATION_SUM_XACT_TIMES
),
181 TokenTableEntry("<last_h", LFT_ADAPTATION_LAST_HEADER
),
182 TokenTableEntry(NULL
, LFT_NONE
) /* this must be last */
187 /// ICAP (icap::) tokens
188 static TokenTableEntry TokenTableIcap
[] = {
189 TokenTableEntry("tt", LFT_ICAP_TOTAL_TIME
),
190 TokenTableEntry("<last_h", LFT_ADAPTATION_LAST_HEADER
), // deprecated
192 TokenTableEntry("<A", LFT_ICAP_ADDR
),
193 TokenTableEntry("<service_name", LFT_ICAP_SERV_NAME
),
194 TokenTableEntry("ru", LFT_ICAP_REQUEST_URI
),
195 TokenTableEntry("rm", LFT_ICAP_REQUEST_METHOD
),
196 TokenTableEntry(">st", LFT_ICAP_BYTES_SENT
),
197 TokenTableEntry("<st", LFT_ICAP_BYTES_READ
),
198 TokenTableEntry("<bs", LFT_ICAP_BODY_BYTES_READ
),
200 TokenTableEntry(">h", LFT_ICAP_REQ_HEADER
),
201 TokenTableEntry("<h", LFT_ICAP_REP_HEADER
),
203 TokenTableEntry("tr", LFT_ICAP_TR_RESPONSE_TIME
),
204 TokenTableEntry("tio", LFT_ICAP_IO_TIME
),
205 TokenTableEntry("to", LFT_ICAP_OUTCOME
),
206 TokenTableEntry("Hs", LFT_ICAP_STATUS_CODE
),
208 TokenTableEntry(NULL
, LFT_NONE
) /* this must be last */
213 // SSL (ssl::) tokens
214 static TokenTableEntry TokenTableSsl
[] = {
215 TokenTableEntry("bump_mode", LFT_SSL_BUMP_MODE
),
216 TokenTableEntry(">cert_subject", LFT_SSL_USER_CERT_SUBJECT
),
217 TokenTableEntry(">cert_issuer", LFT_SSL_USER_CERT_ISSUER
),
218 TokenTableEntry(">sni", LFT_SSL_CLIENT_SNI
),
219 /*TokenTableEntry("<cert_subject", LFT_SSL_SERVER_CERT_SUBJECT), */
220 /*TokenTableEntry("<cert_issuer", LFT_SSL_SERVER_CERT_ISSUER), */
221 TokenTableEntry(NULL
, LFT_NONE
)
224 } // namespace Format
226 /// Register all components custom format tokens
228 Format::Token::Init()
230 // TODO standard log tokens
233 TheConfig
.registerTokens(String("adapt"),::Format::TokenTableAdapt
);
236 TheConfig
.registerTokens(String("icap"),::Format::TokenTableIcap
);
239 TheConfig
.registerTokens(String("ssl"),::Format::TokenTableSsl
);
243 /// Scans a token table to see if the next token exists there
244 /// returns a pointer to next unparsed byte and updates type member if found
246 Format::Token::scanForToken(TokenTableEntry
const table
[], const char *cur
)
248 for (TokenTableEntry
const *lte
= table
; lte
->configTag
!= NULL
; ++lte
) {
249 debugs(46, 8, HERE
<< "compare tokens '" << lte
->configTag
<< "' with '" << cur
<< "'");
250 if (strncmp(lte
->configTag
, cur
, strlen(lte
->configTag
)) == 0) {
251 type
= lte
->tokenType
;
252 label
= lte
->configTag
;
253 debugs(46, 7, HERE
<< "Found token '" << label
<< "'");
254 return cur
+ strlen(lte
->configTag
);
260 /* parses a single token. Returns the token length in characters,
261 * and fills in the lt item with the token information.
262 * def is for sure null-terminated
265 Format::Token::parse(const char *def
, Quoting
*quoting
)
267 const char *cur
= def
;
271 l
= strcspn(cur
, "%");
275 /* it's a string for sure, until \0 or the next % */
276 cp
= (char *)xmalloc(l
+ 1);
277 xstrncpy(cp
, cur
, l
+ 1);
286 if (*quoting
== LOG_QUOTE_NONE
)
287 *quoting
= LOG_QUOTE_QUOTES
;
288 else if (*quoting
== LOG_QUOTE_QUOTES
)
289 *quoting
= LOG_QUOTE_NONE
;
294 if (*quoting
== LOG_QUOTE_NONE
)
295 *quoting
= LOG_QUOTE_MIMEBLOB
;
300 if (*quoting
== LOG_QUOTE_MIMEBLOB
)
301 *quoting
= LOG_QUOTE_NONE
;
314 // select quoting style for his particular token
318 quote
= LOG_QUOTE_QUOTES
;
323 quote
= LOG_QUOTE_RAW
;
328 quote
= LOG_QUOTE_MIMEBLOB
;
333 quote
= LOG_QUOTE_URL
;
353 if (xisdigit(*cur
)) {
354 widthMin
= strtol(cur
, &endp
, 10);
358 if (*cur
== '.' && xisdigit(*(++cur
))) {
359 widthMax
= strtol(cur
, &endp
, 10);
363 // when {arg} field is before the token (old logformat syntax)
367 l
= strcspn(cur
, "}");
368 cp
= (char *)xmalloc(l
+ 1);
369 xstrncpy(cp
, cur
, l
+ 1);
379 // Scan each registered token namespace
380 debugs(46, 9, HERE
<< "check for token in " << TheConfig
.tokens
.size() << " namespaces.");
381 for (std::list
<TokenNamespace
>::const_iterator itr
= TheConfig
.tokens
.begin(); itr
!= TheConfig
.tokens
.end(); ++itr
) {
382 debugs(46, 7, HERE
<< "check for possible " << itr
->prefix
<< ":: token");
383 const size_t len
= itr
->prefix
.size();
384 if (itr
->prefix
.cmp(cur
, len
) == 0 && cur
[len
] == ':' && cur
[len
+1] == ':') {
385 debugs(46, 5, HERE
<< "check for " << itr
->prefix
<< ":: token in '" << cur
<< "'");
386 const char *old
= cur
;
387 cur
= scanForToken(itr
->tokenSet
, cur
+len
+2);
388 if (old
!= cur
) // found
390 else // reset to start of namespace
395 if (type
== LFT_NONE
) {
396 // For upward compatibility, assume "http::" prefix as default prefix
397 // for all log access formatting codes, except those starting with a
398 // "%" or a known namespace. (ie "icap::", "adapt::")
399 if (strncmp(cur
,"http::", 6) == 0 && *(cur
+6) != '%' )
402 // NP: scan the sets of tokens in decreasing size to guarantee no
403 // mistakes made with overlapping names. (Bug 3310)
405 // Scan for various long tokens
406 debugs(46, 5, HERE
<< "scan for possible Misc token");
407 cur
= scanForToken(TokenTableMisc
, cur
);
408 // scan for 2-char tokens
409 if (type
== LFT_NONE
) {
410 debugs(46, 5, HERE
<< "scan for possible 2C token");
411 cur
= scanForToken(TokenTable2C
, cur
);
413 // finally scan for 1-char tokens.
414 if (type
== LFT_NONE
) {
415 debugs(46, 5, HERE
<< "scan for possible 1C token");
416 cur
= scanForToken(TokenTable1C
, cur
);
420 if (type
== LFT_NONE
) {
421 fatalf("Can't parse configuration token: '%s'\n", def
);
424 // when {arg} field is after the token (old external_acl_type token syntax)
425 // but accept only if there was none before the token
426 if (*cur
== '{' && !data
.string
) {
429 l
= strcspn(cur
, "}");
430 cp
= (char *)xmalloc(l
+ 1);
431 xstrncpy(cp
, cur
, l
+ 1);
448 case LFT_ADAPTATION_LAST_HEADER
:
452 case LFT_ICAP_REQ_HEADER
:
454 case LFT_ICAP_REP_HEADER
:
457 case LFT_ADAPTED_REQUEST_HEADER
:
459 case LFT_REQUEST_HEADER
:
461 case LFT_REPLY_HEADER
:
466 char *header
= data
.string
;
467 char *cp
= strchr(header
, ':');
473 if (*cp
== ',' || *cp
== ';' || *cp
== ':') {
474 data
.header
.separator
= *cp
;
477 data
.header
.separator
= ',';
480 data
.header
.element
= cp
;
483 case LFT_REQUEST_HEADER
:
484 type
= LFT_REQUEST_HEADER_ELEM
;
487 case LFT_ADAPTED_REQUEST_HEADER
:
488 type
= LFT_ADAPTED_REQUEST_HEADER_ELEM
;
491 case LFT_REPLY_HEADER
:
492 type
= LFT_REPLY_HEADER_ELEM
;
495 case LFT_ADAPTATION_LAST_HEADER
:
496 type
= LFT_ADAPTATION_LAST_HEADER_ELEM
;
500 case LFT_ICAP_REQ_HEADER
:
501 type
= LFT_ICAP_REQ_HEADER_ELEM
;
503 case LFT_ICAP_REP_HEADER
:
504 type
= LFT_ICAP_REP_HEADER_ELEM
;
512 data
.header
.header
= header
;
515 case LFT_REQUEST_HEADER
:
516 type
= LFT_REQUEST_ALL_HEADERS
;
519 case LFT_ADAPTED_REQUEST_HEADER
:
520 type
= LFT_ADAPTED_REQUEST_ALL_HEADERS
;
523 case LFT_REPLY_HEADER
:
524 type
= LFT_REPLY_ALL_HEADERS
;
527 case LFT_ADAPTATION_LAST_HEADER
:
528 type
= LFT_ADAPTATION_LAST_ALL_HEADERS
;
532 case LFT_ICAP_REQ_HEADER
:
533 type
= LFT_ICAP_REQ_ALL_HEADERS
;
535 case LFT_ICAP_REP_HEADER
:
536 type
= LFT_ICAP_REP_ALL_HEADERS
;
542 Config
.onoff
.log_mime_hdrs
= 1;
547 case LFT_CLIENT_FQDN
:
548 Config
.onoff
.log_fqdn
= 1;
551 case LFT_TIME_TO_HANDLE_REQUEST
:
552 case LFT_PEER_RESPONSE_TIME
:
553 case LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME
:
554 case LFT_DNS_WAIT_TIME
:
556 case LFT_ICAP_TR_RESPONSE_TIME
:
557 case LFT_ICAP_IO_TIME
:
558 case LFT_ICAP_TOTAL_TIME
:
561 case LFT_TIME_SUBSECOND
:
567 for (int i
= widthMax
; i
> 0; --i
)
575 case LFT_HTTP_SENT_STATUS_CODE_OLD_30
:
576 debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT
), "WARNING: The \"Hs\" formatting code is deprecated. Use the \">Hs\" instead.");
577 type
= LFT_HTTP_SENT_STATUS_CODE
;
580 case LFT_SERVER_LOCAL_IP_OLD_27
:
581 debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT
), "WARNING: The \"oa\" formatting code is deprecated. Use the \"<la\" instead.");
582 type
= LFT_SERVER_LOCAL_IP
;
585 case LFT_REQUEST_URLPATH_OLD_31
:
586 debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT
), "WARNING: The \"rp\" formatting code is deprecated. Use the \">rp\" instead.");
587 type
= LFT_CLIENT_REQ_URLPATH
;
590 case LFT_REQUEST_VERSION_OLD_2X
:
591 debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT
), "WARNING: The \">v\" formatting code is deprecated. Use the \">rv\" instead.");
592 type
= LFT_REQUEST_VERSION
;
597 debugs(46, DBG_CRITICAL
, "WARNING: The \">eui\" formatting code requires EUI features which are disabled in this Squid.");
601 case LFT_REQUEST_URLGROUP_OLD_2X
:
602 debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT
), "WARNING: The \"rG\" formatting code is deprecated. Use \"note{urlgroup}\" instead.");
604 data
.header
.header
= xstrdup("urlgroup");
614 Format::Token::Token() : type(LFT_NONE
),
618 quote(LOG_QUOTE_NONE
),
626 data
.header
.header
= NULL
;
627 data
.header
.element
= NULL
;
628 data
.header
.separator
= ',';
631 Format::Token::~Token()
633 label
= NULL
; // drop reference to global static.
634 safe_free(data
.string
);
636 Token
*tokens
= next
;