http: logs custom headers in a subobject

This subobject is request_headers or response_headers

This especially avoids json keys collisions.

Ticket: #5320

Also fixes typo referrer/referer

diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst
index 31f08b54d3e86b9c8929454a1551d24a522ab30c..978d5cbfa24d0f03c2dee870a06e0861d01da0f1 100644 (file)
--- a/doc/userguide/output/eve/eve-json-format.rst
+++ b/doc/userguide/output/eve/eve-json-format.rst
@@ -286,7 +286,7 @@ suricata.yaml file the following fields are (can) also included:
 * "status": HTTP status code
 * "protocol": Protocol / Version of HTTP (ex: HTTP/1.1)
 * "http_method": The HTTP method (ex: GET, POST, HEAD)
-* "http_refer": The referrer for this action
+* "http_refer": The referer for this action
 
 In addition to the extended logging fields one can also choose to enable/add
 from more than 50 additional custom logging HTTP fields enabled in the
@@ -318,7 +318,7 @@ suricata.yaml file. The additional fields can be enabled as following:
               allow, connection, content-encoding, content-language,
               content-length, content-location, content-md5, content-range,
               content-type, date, etags, expires, last-modified, link, location,
-              proxy-authenticate, referrer, refresh, retry-after, server,
+              proxy-authenticate, referer, refresh, retry-after, server,
               set-cookie, trailer, transfer-encoding, upgrade, vary, warning,
               www-authenticate, x-flash-version, x-authenticated-user]
 

diff --git a/doc/userguide/output/eve/eve-json-output.rst b/doc/userguide/output/eve/eve-json-output.rst
index bf288dabcc06f3500990853793fcda32266e2eb8..c6d7ec3951662cf7730f815f4739a68f4dd24b83 100644 (file)
--- a/doc/userguide/output/eve/eve-json-output.rst
+++ b/doc/userguide/output/eve/eve-json-output.rst
@@ -192,7 +192,7 @@ last_modified           last-modified
 link                    link
 location                location
 proxy_authenticate      proxy-authenticate
-referrer                referrer
+referer                 referer
 refresh                 refresh
 retry_after             retry-after
 server                  server

diff --git a/src/output-json-http.c b/src/output-json-http.c
index e12555375bfad5f48acc0d6733b9e2280dd33ff6..30b33c74527fcb44ca80cf128ce41171094ffcb0 100644 (file)
--- a/src/output-json-http.c
+++ b/src/output-json-http.c
@@ -138,7 +138,7 @@ struct {
     const char *config_field;
     const char *htp_field;
     uint32_t flags;
-} http_fields[] =  {
+} http_fields[] = {
     { "accept", "accept", LOG_HTTP_REQUEST },
     { "accept_charset", "accept-charset", LOG_HTTP_REQUEST },
     { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST },
@@ -146,7 +146,7 @@ struct {
     { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST },
     { "authorization", "authorization", LOG_HTTP_REQUEST },
     { "cache_control", "cache-control", LOG_HTTP_REQUEST },
-    { "cookie", "cookie", LOG_HTTP_REQUEST|LOG_HTTP_ARRAY },
+    { "cookie", "cookie", LOG_HTTP_REQUEST | LOG_HTTP_ARRAY },
     { "from", "from", LOG_HTTP_REQUEST },
     { "max_forwards", "max-forwards", LOG_HTTP_REQUEST },
     { "origin", "origin", LOG_HTTP_REQUEST },
@@ -173,12 +173,12 @@ struct {
     { "content_type", "content-type", 0 },
     { "date", "date", 0 },
     { "etag", "etags", 0 },
-    { "expires", "expires" , 0 },
+    { "expires", "expires", 0 },
     { "last_modified", "last-modified", 0 },
     { "link", "link", 0 },
     { "location", "location", 0 },
     { "proxy_authenticate", "proxy-authenticate", 0 },
-    { "referrer", "referrer", LOG_HTTP_EXTENDED },
+    { "referer", "referer", LOG_HTTP_EXTENDED },
     { "refresh", "refresh", 0 },
     { "retry_after", "retry-after", 0 },
     { "server", "server", 0 },
@@ -264,46 +264,6 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx)
     }
 }
 
-static void EveHttpLogJSONCustom(LogHttpFileCtx *http_ctx, JsonBuilder *js, htp_tx_t *tx)
-{
-    char *c;
-    HttpField f;
-
-    for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
-    {
-        if ((http_ctx->fields & (1ULL<<f)) != 0)
-        {
-            /* prevent logging a field twice if extended logging is
-                enabled */
-            if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) ||
-                ((http_ctx->flags & LOG_HTTP_EXTENDED) !=
-                      (http_fields[f].flags & LOG_HTTP_EXTENDED)))
-            {
-                htp_header_t *h_field = NULL;
-                if ((http_fields[f].flags & LOG_HTTP_REQUEST) != 0)
-                {
-                    if (tx->request_headers != NULL) {
-                        h_field = htp_table_get_c(tx->request_headers,
-                                                  http_fields[f].htp_field);
-                    }
-                } else {
-                    if (tx->response_headers != NULL) {
-                        h_field = htp_table_get_c(tx->response_headers,
-                                                  http_fields[f].htp_field);
-                    }
-                }
-                if (h_field != NULL) {
-                    c = bstr_util_strdup_to_c(h_field->value);
-                    if (c != NULL) {
-                        jb_set_string(js, http_fields[f].config_field, c);
-                        SCFree(c);
-                    }
-                }
-            }
-        }
-    }
-}
-
 static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx)
 {
     /* referer */
@@ -348,19 +308,44 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx)
     jb_set_uint(js, "length", tx->response_message_len);
 }
 
-static void EveHttpLogJSONHeaders(JsonBuilder *js, uint32_t direction, htp_tx_t *tx)
+static void EveHttpLogJSONHeaders(
+        JsonBuilder *js, uint32_t direction, htp_tx_t *tx, LogHttpFileCtx *http_ctx)
 {
     htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ?
         tx->request_headers : tx->response_headers;
     char name[MAX_SIZE_HEADER_NAME] = {0};
     char value[MAX_SIZE_HEADER_VALUE] = {0};
     size_t n = htp_table_size(headers);
+    JsonBuilderMark mark = { 0, 0, 0 };
+    jb_get_mark(js, &mark);
+    bool array_empty = true;
     jb_open_array(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers");
     for (size_t i = 0; i < n; i++) {
         htp_header_t * h = htp_table_get_index(headers, i, NULL);
         if (h == NULL) {
             continue;
         }
+        if ((http_ctx->flags & direction) == 0 && http_ctx->fields != 0) {
+            bool tolog = false;
+            for (HttpField f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) {
+                if ((http_ctx->fields & (1ULL << f)) != 0) {
+                    /* prevent logging a field twice if extended logging is
+                     enabled */
+                    if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) ||
+                            ((http_ctx->flags & LOG_HTTP_EXTENDED) !=
+                                    (http_fields[f].flags & LOG_HTTP_EXTENDED))) {
+                        if (bstr_cmp_c_nocase(h->name, http_fields[f].htp_field) == 0) {
+                            tolog = true;
+                            break;
+                        }
+                    }
+                }
+            }
+            if (!tolog) {
+                continue;
+            }
+        }
+        array_empty = false;
         jb_start_object(js);
         size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ?
             bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1;
@@ -374,8 +359,12 @@ static void EveHttpLogJSONHeaders(JsonBuilder *js, uint32_t direction, htp_tx_t
         jb_set_string(js, "value", value);
         jb_close(js);
     }
-    // Close array.
-    jb_close(js);
+    if (array_empty) {
+        jb_restore_mark(js, &mark);
+    } else {
+        // Close array.
+        jb_close(js);
+    }
 }
 
 static void BodyPrintableBuffer(JsonBuilder *js, HtpBody *body, const char *key)
@@ -454,15 +443,12 @@ static void EveHttpLogJSON(JsonHttpLogThread *aft, JsonBuilder *js, htp_tx_t *tx
     jb_open_object(js, "http");
 
     EveHttpLogJSONBasic(js, tx);
-    /* log custom fields if configured */
-    if (http_ctx->fields != 0)
-        EveHttpLogJSONCustom(http_ctx, js, tx);
     if (http_ctx->flags & LOG_HTTP_EXTENDED)
         EveHttpLogJSONExtended(js, tx);
-    if (http_ctx->flags & LOG_HTTP_REQ_HEADERS)
-        EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx);
-    if (http_ctx->flags & LOG_HTTP_RES_HEADERS)
-        EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx);
+    if (http_ctx->flags & LOG_HTTP_REQ_HEADERS || http_ctx->fields != 0)
+        EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx, http_ctx);
+    if (http_ctx->flags & LOG_HTTP_RES_HEADERS || http_ctx->fields != 0)
+        EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx, http_ctx);
 
     jb_close(js);
 }
@@ -566,8 +552,23 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c
             }
         }
 
+        const char *all_headers = ConfNodeLookupChildValue(conf, "dump-all-headers");
+        if (all_headers != NULL) {
+            if (strncmp(all_headers, "both", 4) == 0) {
+                http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
+                http_ctx->flags |= LOG_HTTP_RES_HEADERS;
+            } else if (strncmp(all_headers, "request", 7) == 0) {
+                http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
+            } else if (strncmp(all_headers, "response", 8) == 0) {
+                http_ctx->flags |= LOG_HTTP_RES_HEADERS;
+            }
+        }
         ConfNode *custom;
         if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
+            if ((http_ctx->flags & (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) ==
+                    (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) {
+                SCLogWarning("No need for custom as dump-all-headers is already present");
+            }
             ConfNode *field;
             TAILQ_FOREACH (field, &custom->head, next) {
                 HttpField f;
@@ -580,18 +581,6 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c
                 }
             }
         }
-        const char *all_headers = ConfNodeLookupChildValue(
-                conf, "dump-all-headers");
-        if (all_headers != NULL) {
-            if (strncmp(all_headers, "both", 4) == 0) {
-                http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
-                http_ctx->flags |= LOG_HTTP_RES_HEADERS;
-            } else if (strncmp(all_headers, "request", 7) == 0) {
-                http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
-            } else if (strncmp(all_headers, "response", 8) == 0) {
-                http_ctx->flags |= LOG_HTTP_RES_HEADERS;
-            }
-        }
     }
 
     if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {