src/format/Token.cc

   1 #include "squid.h"
   2 #include "format/Config.h"
   3 #include "format/Token.h"
   4 #include "format/TokenTableEntry.h"
   5 #include "SquidConfig.h"
   6 #include "Store.h"
   7
   8 const char *Format::log_tags[] = {
   9     "NONE",
  10     "TCP_HIT",
  11     "TCP_MISS",
  12     "TCP_REFRESH_UNMODIFIED",
  13     "TCP_REFRESH_FAIL", // same tag logged for LOG_TCP_REFRESH_FAIL_OLD and
  14     "TCP_REFRESH_FAIL", // LOG_TCP_REFRESH_FAIL_ERR for backward-compatibility
  15     "TCP_REFRESH_MODIFIED",
  16     "TCP_CLIENT_REFRESH_MISS",
  17     "TCP_IMS_HIT",
  18     "TCP_SWAPFAIL_MISS",
  19     "TCP_NEGATIVE_HIT",
  20     "TCP_MEM_HIT",
  21     "TCP_DENIED",
  22     "TCP_DENIED_REPLY",
  23     "TCP_OFFLINE_HIT",
  24     "TCP_REDIRECT",
  25     "UDP_HIT",
  26     "UDP_MISS",
  27     "UDP_DENIED",
  28     "UDP_INVALID",
  29     "UDP_MISS_NOFETCH",
  30     "ICP_QUERY",
  31     "LOG_TYPE_MAX"
  32 };
  33
  34 // Due to token overlaps between 1 and 2 letter tokens (Bug 3310)
  35 // We split the token table into sets determined by the token length
  36 namespace Format
  37 {
  38
  39 /// 1-char tokens.
  40 static TokenTableEntry TokenTable1C[] = {
  41
  42     {">a", LFT_CLIENT_IP_ADDRESS},
  43     {">p", LFT_CLIENT_PORT},
  44     {">A", LFT_CLIENT_FQDN},
  45
  46     {"<a", LFT_SERVER_IP_ADDRESS},
  47     {"<p", LFT_SERVER_PORT},
  48     {"<A", LFT_SERVER_FQDN_OR_PEER_NAME},
  49
  50     {">h", LFT_REQUEST_HEADER},
  51     {">h", LFT_REQUEST_ALL_HEADERS},
  52     {"<h", LFT_REPLY_HEADER},
  53     {"<h", LFT_REPLY_ALL_HEADERS},
  54
  55     {">v", LFT_REQUEST_VERSION_OLD_2X},
  56
  57     {"%", LFT_PERCENT},
  58
  59     {NULL, LFT_NONE}            /* this must be last */
  60 };
  61
  62 /// 2-char tokens
  63 static TokenTableEntry TokenTable2C[] = {
  64
  65     {">la", LFT_CLIENT_LOCAL_IP},
  66     {"la", LFT_LOCAL_LISTENING_IP},
  67     {">lp", LFT_CLIENT_LOCAL_PORT},
  68     {"lp", LFT_LOCAL_LISTENING_PORT},
  69     /*{ "lA", LFT_LOCAL_NAME }, */
  70
  71     {"<la", LFT_SERVER_LOCAL_IP},
  72     {"oa", LFT_SERVER_LOCAL_IP_OLD_27},
  73     {"<lp", LFT_SERVER_LOCAL_PORT},
  74     /* {"ot", LFT_PEER_OUTGOING_TOS}, */
  75
  76     {"ts", LFT_TIME_SECONDS_SINCE_EPOCH},
  77     {"tu", LFT_TIME_SUBSECOND},
  78     {"tl", LFT_TIME_LOCALTIME},
  79     {"tg", LFT_TIME_GMT},
  80     {"tr", LFT_TIME_TO_HANDLE_REQUEST},
  81
  82     {"<pt", LFT_PEER_RESPONSE_TIME},
  83     {"<tt", LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME},
  84     {"dt", LFT_DNS_WAIT_TIME},
  85
  86     {">ha", LFT_ADAPTED_REQUEST_HEADER},
  87     {">ha", LFT_ADAPTED_REQUEST_ALL_HEADERS},
  88
  89     {"un", LFT_USER_NAME},
  90     {"ul", LFT_USER_LOGIN},
  91     /*{ "ur", LFT_USER_REALM }, */
  92     /*{ "us", LFT_USER_SCHEME }, */
  93     {"ui", LFT_USER_IDENT},
  94     {"ue", LFT_USER_EXTERNAL},
  95
  96     {"Hs", LFT_HTTP_SENT_STATUS_CODE_OLD_30},
  97     {">Hs", LFT_HTTP_SENT_STATUS_CODE},
  98     {"<Hs", LFT_HTTP_RECEIVED_STATUS_CODE},
  99     /*{ "Ht", LFT_HTTP_STATUS }, */
 100     {"<bs", LFT_HTTP_BODY_BYTES_READ},
 101
 102     {"Ss", LFT_SQUID_STATUS},
 103     {"Sh", LFT_SQUID_HIERARCHY},
 104
 105     {"mt", LFT_MIME_TYPE},
 106
 107     {">rm", LFT_CLIENT_REQ_METHOD},
 108     {">ru", LFT_CLIENT_REQ_URI},
 109     {">rp", LFT_CLIENT_REQ_URLPATH},
 110     /*{">rq", LFT_CLIENT_REQ_QUERY},*/
 111     {">rv", LFT_CLIENT_REQ_VERSION},
 112
 113     {"rm", LFT_REQUEST_METHOD},
 114     {"ru", LFT_REQUEST_URI},    /* doesn't include the query-string */
 115     {"rp", LFT_REQUEST_URLPATH_OLD_31},
 116     /* { "rq", LFT_REQUEST_QUERY }, * /     / * the query-string, INCLUDING the leading ? */
 117     {"rv", LFT_REQUEST_VERSION},
 118
 119     {"<rm", LFT_SERVER_REQ_METHOD},
 120     {"<ru", LFT_SERVER_REQ_URI},
 121     {"<rp", LFT_SERVER_REQ_URLPATH},
 122     /*{"<rq", LFT_SERVER_REQ_QUERY},*/
 123     {"<rv", LFT_SERVER_REQ_VERSION},
 124
 125     {">st", LFT_REQUEST_SIZE_TOTAL },
 126     /*{ ">sl", LFT_REQUEST_SIZE_LINE }, * / / * the request line "GET ... " */
 127     {">sh", LFT_REQUEST_SIZE_HEADERS },
 128     /*{ ">sb", LFT_REQUEST_SIZE_BODY }, */
 129     /*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */
 130
 131     {"<st", LFT_REPLY_SIZE_TOTAL},
 132     {"<sH", LFT_REPLY_HIGHOFFSET},
 133     {"<sS", LFT_REPLY_OBJECTSIZE},
 134     /*{ "<sl", LFT_REPLY_SIZE_LINE }, * /   / * the reply line (protocol, code, text) */
 135     {"<sh", LFT_REPLY_SIZE_HEADERS },
 136     /*{ "<sb", LFT_REPLY_SIZE_BODY }, */
 137     /*{ "<sB", LFT_REPLY_SIZE_BODY_NO_TE }, */
 138
 139     {"et", LFT_TAG},
 140     {"st", LFT_IO_SIZE_TOTAL},
 141     {"ea", LFT_EXT_LOG},
 142     {"sn", LFT_SEQUENCE_NUMBER},
 143
 144     {NULL, LFT_NONE}            /* this must be last */
 145 };
 146
 147 /// Miscellaneous >2 byte tokens
 148 static TokenTableEntry TokenTableMisc[] = {
 149     {">eui", LFT_CLIENT_EUI},
 150     {"err_code", LFT_SQUID_ERROR },
 151     {"err_detail", LFT_SQUID_ERROR_DETAIL },
 152     {NULL, LFT_NONE}            /* this must be last */
 153 };
 154
 155 #if USE_ADAPTATION
 156 static TokenTableEntry TokenTableAdapt[] = {
 157     {"all_trs", LFT_ADAPTATION_ALL_XACT_TIMES},
 158     {"sum_trs", LFT_ADAPTATION_SUM_XACT_TIMES},
 159     {"<last_h", LFT_ADAPTATION_LAST_HEADER},
 160     {NULL, LFT_NONE}           /* this must be last */
 161 };
 162 #endif
 163
 164 #if ICAP_CLIENT
 165 /// ICAP (icap::) tokens
 166 static TokenTableEntry TokenTableIcap[] = {
 167     {"tt", LFT_ICAP_TOTAL_TIME},
 168     {"<last_h", LFT_ADAPTATION_LAST_HEADER}, // deprecated
 169
 170     {"<A",  LFT_ICAP_ADDR},
 171     {"<service_name",  LFT_ICAP_SERV_NAME},
 172     {"ru",  LFT_ICAP_REQUEST_URI},
 173     {"rm",  LFT_ICAP_REQUEST_METHOD},
 174     {">st", LFT_ICAP_BYTES_SENT},
 175     {"<st", LFT_ICAP_BYTES_READ},
 176     {"<bs", LFT_ICAP_BODY_BYTES_READ},
 177
 178     {">h",  LFT_ICAP_REQ_HEADER},
 179     {"<h",  LFT_ICAP_REP_HEADER},
 180
 181     {"tr",  LFT_ICAP_TR_RESPONSE_TIME},
 182     {"tio", LFT_ICAP_IO_TIME},
 183     {"to",  LFT_ICAP_OUTCOME},
 184     {"Hs",  LFT_ICAP_STATUS_CODE},
 185
 186     {NULL, LFT_NONE}           /* this must be last */
 187 };
 188 #endif
 189
 190 #if USE_SSL
 191 // SSL (ssl::) tokens
 192 static TokenTableEntry TokenTableSsl[] = {
 193     {"bump_mode", LFT_SSL_BUMP_MODE},
 194     {">cert_subject", LFT_SSL_USER_CERT_SUBJECT},
 195     {">cert_issuer", LFT_SSL_USER_CERT_ISSUER},
 196     {NULL, LFT_NONE}
 197 };
 198 #endif
 199 } // namespace Format
 200
 201 /// Register all components custom format tokens
 202 void
 203 Format::Token::Init()
 204 {
 205     // TODO standard log tokens
 206     // TODO external ACL fmt tokens
 207
 208 #if USE_ADAPTATION
 209     TheConfig.registerTokens(String("adapt"),::Format::TokenTableAdapt);
 210 #endif
 211 #if ICAP_CLIENT
 212     TheConfig.registerTokens(String("icap"),::Format::TokenTableIcap);
 213 #endif
 214 #if USE_SSL
 215     TheConfig.registerTokens(String("ssl"),::Format::TokenTableSsl);
 216 #endif
 217 }
 218
 219 /// Scans a token table to see if the next token exists there
 220 /// returns a pointer to next unparsed byte and updates type member if found
 221 const char *
 222 Format::Token::scanForToken(TokenTableEntry const table[], const char *cur)
 223 {
 224     for (TokenTableEntry const *lte = table; lte->configTag != NULL; ++lte) {
 225         debugs(46, 8, HERE << "compare tokens '" << lte->configTag << "' with '" << cur << "'");
 226         if (strncmp(lte->configTag, cur, strlen(lte->configTag)) == 0) {
 227             type = lte->tokenType;
 228             label = lte->configTag;
 229             debugs(46, 7, HERE << "Found token '" << label << "'");
 230             return cur + strlen(lte->configTag);
 231         }
 232     }
 233     return cur;
 234 }
 235
 236 /* parses a single token. Returns the token length in characters,
 237  * and fills in the lt item with the token information.
 238  * def is for sure null-terminated
 239  */
 240 int
 241 Format::Token::parse(const char *def, Quoting *quoting)
 242 {
 243     const char *cur = def;
 244
 245     int l;
 246
 247     l = strcspn(cur, "%");
 248
 249     if (l > 0) {
 250         char *cp;
 251         /* it's a string for sure, until \0 or the next % */
 252         cp = (char *)xmalloc(l + 1);
 253         xstrncpy(cp, cur, l + 1);
 254         type = LFT_STRING;
 255         data.string = cp;
 256
 257         while (l > 0) {
 258             switch (*cur) {
 259
 260             case '"':
 261
 262                 if (*quoting == LOG_QUOTE_NONE)
 263                     *quoting = LOG_QUOTE_QUOTES;
 264                 else if (*quoting == LOG_QUOTE_QUOTES)
 265                     *quoting = LOG_QUOTE_NONE;
 266
 267                 break;
 268
 269             case '[':
 270                 if (*quoting == LOG_QUOTE_NONE)
 271                     *quoting = LOG_QUOTE_MIMEBLOB;
 272
 273                 break;
 274
 275             case ']':
 276                 if (*quoting == LOG_QUOTE_MIMEBLOB)
 277                     *quoting = LOG_QUOTE_NONE;
 278
 279                 break;
 280             }
 281
 282             ++cur;
 283             --l;
 284         }
 285
 286         goto done;
 287     }
 288
 289     if (!*cur)
 290         goto done;
 291
 292     ++cur;
 293
 294     // select quoting style for his particular token
 295     switch (*cur) {
 296
 297     case '"':
 298         quote = LOG_QUOTE_QUOTES;
 299         ++cur;
 300         break;
 301
 302     case '\'':
 303         quote = LOG_QUOTE_RAW;
 304         ++cur;
 305         break;
 306
 307     case '[':
 308         quote = LOG_QUOTE_MIMEBLOB;
 309         ++cur;
 310         break;
 311
 312     case '#':
 313         quote = LOG_QUOTE_URL;
 314         ++cur;
 315         break;
 316
 317     default:
 318         quote = *quoting;
 319         break;
 320     }
 321
 322     if (*cur == '-') {
 323         left = 1;
 324         ++cur;
 325     }
 326
 327     if (*cur == '0') {
 328         zero = 1;
 329         ++cur;
 330     }
 331
 332     char *endp;
 333     if (xisdigit(*cur)) {
 334         widthMin = strtol(cur, &endp, 10);
 335         cur = endp;
 336     }
 337
 338     if (*cur == '.' && xisdigit(*(++cur))) {
 339         widthMax = strtol(cur, &endp, 10);
 340         cur = endp;
 341     }
 342
 343     if (*cur == '{') {
 344         char *cp;
 345         ++cur;
 346         l = strcspn(cur, "}");
 347         cp = (char *)xmalloc(l + 1);
 348         xstrncpy(cp, cur, l + 1);
 349         data.string = cp;
 350         cur += l;
 351
 352         if (*cur == '}')
 353             ++cur;
 354     }
 355
 356     type = LFT_NONE;
 357
 358     // Scan each registered token namespace
 359     debugs(46, 9, HERE << "check for token in " << TheConfig.tokens.size() << " namespaces.");
 360     for (std::list<TokenNamespace>::const_iterator itr = TheConfig.tokens.begin(); itr != TheConfig.tokens.end(); ++itr) {
 361         debugs(46, 7, HERE << "check for possible " << itr->prefix << ":: token");
 362         const size_t len = itr->prefix.size();
 363         if (itr->prefix.cmp(cur, len) == 0 && cur[len] == ':' && cur[len+1] == ':') {
 364             debugs(46, 5, HERE << "check for " << itr->prefix << ":: token in '" << cur << "'");
 365             const char *old = cur;
 366             cur = scanForToken(itr->tokenSet, cur+len+2);
 367             if (old != cur) // found
 368                 break;
 369             else // reset to start of namespace
 370                 cur = cur - len - 2;
 371         }
 372     }
 373
 374     if (type == LFT_NONE) {
 375         // For upward compatibility, assume "http::" prefix as default prefix
 376         // for all log access formatting codes, except those starting with a
 377         // "%" or a known namespace. (ie "icap::", "adapt::")
 378         if (strncmp(cur,"http::", 6) == 0 && *(cur+6) != '%' )
 379             cur += 6;
 380
 381         // NP: scan the sets of tokens in decreasing size to guarantee no
 382         //     mistakes made with overlapping names. (Bug 3310)
 383
 384         // Scan for various long tokens
 385         debugs(46, 5, HERE << "scan for possible Misc token");
 386         cur = scanForToken(TokenTableMisc, cur);
 387         // scan for 2-char tokens
 388         if (type == LFT_NONE) {
 389             debugs(46, 5, HERE << "scan for possible 2C token");
 390             cur = scanForToken(TokenTable2C, cur);
 391         }
 392         // finally scan for 1-char tokens.
 393         if (type == LFT_NONE) {
 394             debugs(46, 5, HERE << "scan for possible 1C token");
 395             cur = scanForToken(TokenTable1C, cur);
 396         }
 397     }
 398
 399     if (type == LFT_NONE) {
 400         fatalf("Can't parse configuration token: '%s'\n", def);
 401     }
 402
 403     if (*cur == ' ') {
 404         space = 1;
 405         ++cur;
 406     }
 407
 408 done:
 409
 410     switch (type) {
 411
 412 #if USE_ADAPTATION
 413     case LFT_ADAPTATION_LAST_HEADER:
 414 #endif
 415
 416 #if ICAP_CLIENT
 417     case LFT_ICAP_REQ_HEADER:
 418
 419     case LFT_ICAP_REP_HEADER:
 420 #endif
 421
 422     case LFT_ADAPTED_REQUEST_HEADER:
 423
 424     case LFT_REQUEST_HEADER:
 425
 426     case LFT_REPLY_HEADER:
 427
 428         if (data.string) {
 429             char *header = data.string;
 430             char *cp = strchr(header, ':');
 431
 432             if (cp) {
 433                 *cp = '\0';
 434                 ++cp;
 435
 436                 if (*cp == ',' || *cp == ';' || *cp == ':') {
 437                     data.header.separator = *cp;
 438                     ++cp;
 439                 } else {
 440                     data.header.separator = ',';
 441                 }
 442
 443                 data.header.element = cp;
 444
 445                 switch (type) {
 446                 case LFT_REQUEST_HEADER:
 447                     type = LFT_REQUEST_HEADER_ELEM;
 448                     break;
 449
 450                 case LFT_ADAPTED_REQUEST_HEADER:
 451                     type = LFT_ADAPTED_REQUEST_HEADER_ELEM;
 452                     break;
 453
 454                 case LFT_REPLY_HEADER:
 455                     type = LFT_REPLY_HEADER_ELEM;
 456                     break;
 457 #if USE_ADAPTATION
 458                 case LFT_ADAPTATION_LAST_HEADER:
 459                     type = LFT_ADAPTATION_LAST_HEADER_ELEM;
 460                     break;
 461 #endif
 462 #if ICAP_CLIENT
 463                 case LFT_ICAP_REQ_HEADER:
 464                     type = LFT_ICAP_REQ_HEADER_ELEM;
 465                     break;
 466                 case LFT_ICAP_REP_HEADER:
 467                     type = LFT_ICAP_REP_HEADER_ELEM;
 468                     break;
 469 #endif
 470                 default:
 471                     break;
 472                 }
 473             }
 474
 475             data.header.header = header;
 476         } else {
 477             switch (type) {
 478             case LFT_REQUEST_HEADER:
 479                 type = LFT_REQUEST_ALL_HEADERS;
 480                 break;
 481
 482             case LFT_ADAPTED_REQUEST_HEADER:
 483                 type = LFT_ADAPTED_REQUEST_ALL_HEADERS;
 484                 break;
 485
 486             case LFT_REPLY_HEADER:
 487                 type = LFT_REPLY_ALL_HEADERS;
 488                 break;
 489 #if USE_ADAPTATION
 490             case LFT_ADAPTATION_LAST_HEADER:
 491                 type = LFT_ADAPTATION_LAST_ALL_HEADERS;
 492                 break;
 493 #endif
 494 #if ICAP_CLIENT
 495             case LFT_ICAP_REQ_HEADER:
 496                 type = LFT_ICAP_REQ_ALL_HEADERS;
 497                 break;
 498             case LFT_ICAP_REP_HEADER:
 499                 type = LFT_ICAP_REP_ALL_HEADERS;
 500                 break;
 501 #endif
 502             default:
 503                 break;
 504             }
 505             Config.onoff.log_mime_hdrs = 1;
 506         }
 507
 508         break;
 509
 510     case LFT_CLIENT_FQDN:
 511         Config.onoff.log_fqdn = 1;
 512         break;
 513
 514     case LFT_TIME_SUBSECOND:
 515         divisor = 1000;
 516
 517         if (widthMax > 0) {
 518             int i;
 519             divisor = 1000000;
 520
 521             for (i = widthMax; i > 1; --i)
 522                 divisor /= 10;
 523
 524             if (!divisor)
 525                 divisor = 0;
 526         }
 527         break;
 528
 529     case LFT_HTTP_SENT_STATUS_CODE_OLD_30:
 530         debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: The \"Hs\" formatting code is deprecated. Use the \">Hs\" instead.");
 531         type = LFT_HTTP_SENT_STATUS_CODE;
 532         break;
 533
 534     case LFT_SERVER_LOCAL_IP_OLD_27:
 535         debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: The \"oa\" formatting code is deprecated. Use the \"<la\" instead.");
 536         type = LFT_SERVER_LOCAL_IP;
 537         break;
 538
 539     case LFT_REQUEST_URLPATH_OLD_31:
 540         debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: The \"rp\" formatting code is deprecated. Use the \">rp\" instead.");
 541         type = LFT_CLIENT_REQ_URLPATH;
 542         break;
 543
 544     case LFT_REQUEST_VERSION_OLD_2X:
 545         debugs(46, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: The \">v\" formatting code is deprecated. Use the \">rv\" instead.");
 546         type = LFT_REQUEST_VERSION;
 547         break;
 548
 549 #if !USE_SQUID_EUI
 550     case LFT_CLIENT_EUI:
 551         debugs(46, DBG_CRITICAL, "WARNING: The \">eui\" formatting code requires EUI features which are disabled in this Squid.");
 552         break;
 553 #endif
 554
 555     default:
 556         break;
 557     }
 558
 559     return (cur - def);
 560 }
 561
 562 Format::Token::~Token()
 563 {
 564     label = NULL; // drop reference to global static.
 565     safe_free(data.string);
 566     while (next) {
 567         Token *tokens = next;
 568         next = next->next;
 569         tokens->next = NULL;
 570         delete tokens;
 571     }
 572 }
 573