Request-Line log format codes for split client and server view

Since URL-rewrite, adaptation and a number of other things can alter the
request as it goes through Squid it is useful to be able to log both the
original incoming and the final outgoing versions of these details.

This adds logformat codes to extend %rm %ru %rp and %rv with < and >
modifiers for in and outbound display.

The old tag versions are kept for now. There is still work to do cleaning
up the log data object which will clarify if the various data fields
which used to be passed to logging match either of the original or final
request data and can be obsoleted. It is already clear that the logged
path matches neither and contains some normalized version.

diff --git a/src/cf.data.pre b/src/cf.data.pre
index 96663996042ae8fa0a9b4f35603ae08172f0074d..1e7caaae28ebb30bc9a069798b748a5898aaf715 100644 (file)
--- a/src/cf.data.pre
+++ b/src/cf.data.pre
@@ -2911,9 +2911,17 @@ DOC_START
                [http::]Sh      Squid hierarchy status (DEFAULT_PARENT etc)
                [http::]mt      MIME content type
                [http::]rm      Request method (GET/POST etc)
-               [http::]ru      Request URL
+               [http::]>rm     Request method from client
+               [http::]<rm     Request method sent to server or peer
+               [http::]ru      Request URL from client (historic, filtered for logging)
+               [http::]>ru     Request URL from client
+               [http::]<ru     Request URL sent to server or peer
                [http::]rp      Request URL-Path excluding hostname
+               [http::]>rp     Request URL-Path excluding hostname from client
+               [http::]<rp     Request URL-Path excluding hostname sento to server or peer
                [http::]rv      Request protocol version
+               [http::]>rv     Request protocol version from client
+               [http::]<rv     Request protocol version sent to server or peer
                [http::]et      Tag returned by external acl
                [http::]ea      Log string returned by external acl
                [http::]<st     Sent reply size including HTTP headers

diff --git a/src/log/FormatSquidCustom.cc b/src/log/FormatSquidCustom.cc
index 46102f7330e8cc6a93841fae2a142508c1bc5482..d5250b42b3609dd5947e8e1e75805923e6591f12 100644 (file)
--- a/src/log/FormatSquidCustom.cc
+++ b/src/log/FormatSquidCustom.cc
@@ -655,28 +655,81 @@ Log::Format::SquidCustom(AccessLogEntry * al, customlog * log)
 
             break;
 
-        case LFT_REQUEST_METHOD:
-            out = al->_private.method_str;
-
+        case LFT_CLIENT_REQ_METHOD:
+            if (al->request) {
+                out = al->request->method.image();
+                quote = 1;
+            }
             break;
 
-        case LFT_REQUEST_URI:
-            out = al->url;
-
+        case LFT_CLIENT_REQ_URI:
+            // original client URI
+            if (al->request) {
+                out = urlCanonical(al->request);
+                quote = 1;
+            }
             break;
 
-        case LFT_REQUEST_URLPATH:
+        case LFT_REQUEST_URLPATH_OLD_31:
+        case LFT_CLIENT_REQ_URLPATH:
             if (al->request) {
                 out = al->request->urlpath.termedBuf();
                 quote = 1;
             }
             break;
 
+        case LFT_CLIENT_REQ_VERSION:
+            if (al->request) {
+                snprintf(tmp, sizeof(tmp), "%d.%d", (int) al->request->http_ver.major, (int) al->request->http_ver.minor);
+                out = tmp;
+            }
+            break;
+
+        case LFT_REQUEST_METHOD:
+            out = al->_private.method_str;
+            break;
+
+        case LFT_REQUEST_URI:
+            out = al->url;
+            break;
+
+        case LFT_REQUEST_VERSION_OLD_2X:
         case LFT_REQUEST_VERSION:
             snprintf(tmp, sizeof(tmp), "%d.%d", (int) al->http.version.major, (int) al->http.version.minor);
             out = tmp;
             break;
 
+        case LFT_SERVER_REQ_METHOD:
+            if (al->adapted_request) {
+                out = al->adapted_request->method.image();
+                quote = 1;
+            }
+            break;
+
+        case LFT_SERVER_REQ_URI:
+            // adapted request URI sent to server/peer
+            if (al->adapted_request) {
+                out = urlCanonical(al->adapted_request);
+                quote = 1;
+            }
+            break;
+
+        case LFT_SERVER_REQ_URLPATH:
+            if (al->adapted_request) {
+                out = al->adapted_request->urlpath.termedBuf();
+                quote = 1;
+            }
+            break;
+
+        case LFT_SERVER_REQ_VERSION:
+            if (al->adapted_request) {
+                snprintf(tmp, sizeof(tmp), "%d.%d",
+                         (int) al->adapted_request->http_ver.major,
+                         (int) al->adapted_request->http_ver.minor);
+                out = tmp;
+            }
+            break;
+
         case LFT_REQUEST_SIZE_TOTAL:
             outoff = al->cache.requestSize;
             dooff = 1;

diff --git a/src/log/Tokens.cc b/src/log/Tokens.cc
index 438e75b67b6b90ed081a6fd1d8c39c5820e6543e..76c2507e021e73da445283ac9120963dc2b7b598 100644 (file)
--- a/src/log/Tokens.cc
+++ b/src/log/Tokens.cc
@@ -131,13 +131,25 @@ struct logformat_token_table_entry logformat_token_table[] = {
 
     {"mt", LFT_MIME_TYPE},
 
+    {">rm", LFT_CLIENT_REQ_METHOD},
+    {">ru", LFT_CLIENT_REQ_URI},
+    {">rp", LFT_CLIENT_REQ_URLPATH},
+    /*{">rq", LFT_CLIENT_REQ_QUERY},*/
+    {">rv", LFT_CLIENT_REQ_VERSION},
+
     {"rm", LFT_REQUEST_METHOD},
     {"ru", LFT_REQUEST_URI},   /* doesn't include the query-string */
-    {"rp", LFT_REQUEST_URLPATH},       /* doesn't include the host */
+    {"rp", LFT_REQUEST_URLPATH_OLD_31},
     /* { "rq", LFT_REQUEST_QUERY }, * /     / * the query-string, INCLUDING the leading ? */
-    {">v", LFT_REQUEST_VERSION},
+    {">v", LFT_REQUEST_VERSION_OLD_2X},
     {"rv", LFT_REQUEST_VERSION},
 
+    {"<rm", LFT_SERVER_REQ_METHOD},
+    {"<ru", LFT_SERVER_REQ_URI},
+    {"<rp", LFT_SERVER_REQ_URLPATH},
+    /*{"<rq", LFT_SERVER_REQ_QUERY},*/
+    {"<rv", LFT_SERVER_REQ_VERSION},
+
     { ">st", LFT_REQUEST_SIZE_TOTAL },
     /*{ ">sl", LFT_REQUEST_SIZE_LINE }, * / / * the request line "GET ... " */
     { ">sh", LFT_REQUEST_SIZE_HEADERS },
@@ -463,6 +475,16 @@ done:
         lt->type = LFT_PEER_LOCAL_IP;
         break;
 
+    case LFT_REQUEST_URLPATH_OLD_31:
+        debugs(46, 0, "WARNING: The \"rp\" formatting code is deprecated. Use the \">rp\" instead.");
+        lt->type = LFT_CLIENT_REQ_URLPATH;
+        break;
+
+    case LFT_REQUEST_VERSION_OLD_2X:
+        debugs(46, 0, "WARNING: The \">v\" formatting code is deprecated. Use the \">rv\" instead.");
+        lt->type = LFT_REQUEST_VERSION;
+        break;
+
     default:
         break;
     }

diff --git a/src/log/Tokens.h b/src/log/Tokens.h
index 7f3a0e73552d8ffb1e29449b68d3a5a257119b6d..1571550a901e3d672a9d70fda7bfadc399199f50 100644 (file)
--- a/src/log/Tokens.h
+++ b/src/log/Tokens.h
@@ -105,12 +105,28 @@ typedef enum {
 
     LFT_MIME_TYPE,
 
+    /* original Request-Line details receved from client */
+    LFT_CLIENT_REQ_METHOD,
+    LFT_CLIENT_REQ_URI,
+    LFT_CLIENT_REQ_URLPATH,
+    /* LFT_CLIENT_REQ_QUERY, */
+    LFT_CLIENT_REQ_VERSION,
+
+    /* Request-Line details receved from client (legacy, filtered) */
     LFT_REQUEST_METHOD,
     LFT_REQUEST_URI,
-    LFT_REQUEST_URLPATH,
-    /*LFT_REQUEST_QUERY, * // * this is not needed. see strip_query_terms */
+    LFT_REQUEST_URLPATH_OLD_31,
+    /*LFT_REQUEST_QUERY, */
+    LFT_REQUEST_VERSION_OLD_2X,
     LFT_REQUEST_VERSION,
 
+    /* Request-Line details sent to the server/peer */
+    LFT_SERVER_REQ_METHOD,
+    LFT_SERVER_REQ_URI,
+    LFT_SERVER_REQ_URLPATH,
+    /*LFT_SERVER_REQ_QUERY, */
+    LFT_SERVER_REQ_VERSION,
+
     LFT_REQUEST_SIZE_TOTAL,
     /*LFT_REQUEST_SIZE_LINE, */
     LFT_REQUEST_SIZE_HEADERS,