From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Thu, 6 Jul 2017 20:48:18 +0000 (-0400)
Subject: Merge pull request #945 in SNORT/snort3 from nhttp79 to master
X-Git-Tag: 3.0.0-239~28
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=21b386aee608966254347b58df3b2d1c24ccd963;p=thirdparty%2Fsnort3.git

Merge pull request #945 in SNORT/snort3 from nhttp79 to master

Squashed commit of the following:

commit a192d033425753f515840f82ba6413e07f035c8a
Author: Tom Peters <thopeter@cisco.com>
Date:   Tue Jun 27 12:22:15 2017 -0400

    Header normalization improvements
---

diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h
index 9e94e6e27..4424fc066 100644
--- a/src/service_inspectors/http_inspect/http_enum.h
+++ b/src/service_inspectors/http_inspect/http_enum.h
@@ -122,12 +122,13 @@ enum HeaderId { HEAD__NOT_COMPUTE=-14, HEAD__PROBLEMATIC=-12, HEAD__NOT_PRESENT=
     HEAD_WWW_AUTHENTICATE, HEAD_ALLOW, HEAD_CONTENT_ENCODING, HEAD_CONTENT_LANGUAGE,
     HEAD_CONTENT_LENGTH, HEAD_CONTENT_LOCATION, HEAD_CONTENT_MD5, HEAD_CONTENT_RANGE,
     HEAD_CONTENT_TYPE, HEAD_EXPIRES, HEAD_LAST_MODIFIED, HEAD_X_FORWARDED_FOR, HEAD_TRUE_CLIENT_IP,
-    HEAD_X_WORKING_WITH, HEAD_CONTENT_TRANSFER_ENCODING,
+    HEAD_X_WORKING_WITH, HEAD_CONTENT_TRANSFER_ENCODING, HEAD_MIME_VERSION,
     HEAD__MAX_VALUE };
 
 // All the infractions we might find while parsing and analyzing a message
 enum Infraction
 {
+    INF__NONE = -1,
     INF_BARE_BYTE = 0,
     INF_HEAD_TOO_LONG,
     INF_BAD_REQ_LINE,
@@ -226,6 +227,7 @@ enum Infraction
     INF_CHUNKED_ONE_POINT_ZERO,
     INF_CTE_HEADER,
     INF_ILLEGAL_TRAILER,
+    INF_REPEATED_HEADER,
     INF__MAX_VALUE
 };
 
@@ -239,6 +241,7 @@ enum Contentcoding { CONTENTCODE__OTHER=1, CONTENTCODE_GZIP, CONTENTCODE_DEFLATE
 
 enum EventSid
 {
+    EVENT__NONE = -1,
     EVENT_ASCII = 1,
     EVENT_DOUBLE_DECODE,
     EVENT_U_ENCODE,
@@ -332,6 +335,7 @@ enum EventSid
     EVENT_CHUNKED_ONE_POINT_ZERO,
     EVENT_CTE_HEADER,
     EVENT_ILLEGAL_TRAILER,
+    EVENT_REPEATED_HEADER,
     EVENT__MAX_VALUE
 };
 
diff --git a/src/service_inspectors/http_inspect/http_event_gen.h b/src/service_inspectors/http_inspect/http_event_gen.h
index abeef6830..944b364be 100644
--- a/src/service_inspectors/http_inspect/http_event_gen.h
+++ b/src/service_inspectors/http_inspect/http_event_gen.h
@@ -40,6 +40,8 @@ public:
 
     virtual void create_event(int sid)
     {
+        if (sid == HttpEnums::EVENT__NONE)
+            return;
         assert((sid > 0) && (sid <= MAX));
         if (!events_generated[sid-1])
         {
diff --git a/src/service_inspectors/http_inspect/http_header_normalizer.cc b/src/service_inspectors/http_inspect/http_header_normalizer.cc
index b77465a47..fc6ce9eaa 100644
--- a/src/service_inspectors/http_inspect/http_header_normalizer.cc
+++ b/src/service_inspectors/http_inspect/http_header_normalizer.cc
@@ -66,10 +66,13 @@ void HeaderNormalizer::normalize(const HeaderId head_id, const int count,
 
     assert(count > 0);
 
-    // Search Header IDs from all the headers in this message. concatenate_repeats means the header
-    // can properly be present more than once. The standard normalization is to concatenate all the
-    // repeated field values into a comma-separated list. Otherwise only the first value will be
-    // normalized and the rest will be ignored.
+    // Search Header IDs from all the headers in this message. All repeated field values are
+    // concatenated into a comma-separated list.
+    // FIXIT-L Set-Cookie is a special case in the RFC because multiple Set-Cookie headers are
+    // widely used but comma-concatenation of cookies is incorrect. That would be a concern for us
+    // if we actually used the cookies. But since we just want a single value to show to the
+    // pattern matcher, concatenating is probably fine. In the future we may wish to revisit this
+    // issue. Specifically, semicolon-concatenation may be better.
     int num_matches = 0;
     int32_t buffer_length = 0;
 
@@ -82,12 +85,11 @@ void HeaderNormalizer::normalize(const HeaderId head_id, const int count,
             if (++num_matches == 1)
                 curr_match = k;   // remembering location of the first matching header
             buffer_length += header_value[k].length();
-            if (!concatenate_repeats || (num_matches >= count))
+            if (num_matches >= count)
                 break;
         }
     }
-    assert((!concatenate_repeats && (num_matches == 1)) ||
-            (concatenate_repeats && (num_matches == count)));
+    assert(num_matches == count);
     buffer_length += num_matches - 1;    // allow space for concatenation commas
 
     // We are allocating two buffers to store the normalized field value. The raw field value will
@@ -101,7 +103,8 @@ void HeaderNormalizer::normalize(const HeaderId head_id, const int count,
     uint8_t* const temp_space = new uint8_t[buffer_length];
     memset(norm_value, 0, buffer_length);
     memset(temp_space, 0, buffer_length);
-    uint8_t* working = (num_normalizers%2 == 0) ? norm_value : temp_space;
+    uint8_t* const norm_start = (num_normalizers%2 == 0) ? norm_value : temp_space;
+    uint8_t* working = norm_start;
     int32_t data_length = 0;
     for (int j=0; j < num_matches; j++)
     {
@@ -117,6 +120,31 @@ void HeaderNormalizer::normalize(const HeaderId head_id, const int count,
         data_length += growth;
     }
 
+    // Many fields names can appear more than once but some should not. If an event or infraction
+    // is defined we will check as part of normalization. A comma-separated header value is
+    // equivalent to a repeated header name. This is JIT code and we will not check for repeated
+    // headers unless someone asks for that header.
+    if ((repeat_event != EVENT__NONE) || (repeat_inf != INF__NONE))
+    {
+        if (count >= 2)
+        {
+            *infractions += repeat_inf;
+            events->create_event(repeat_event);
+        }
+        else
+        {
+            for (int k=0; k < data_length; k++)
+            {
+                if (norm_start[k] == ',')
+                {
+                    *infractions += repeat_inf;
+                    events->create_event(repeat_event);
+                    break;
+                }
+            }
+        }
+    }
+
     for (int i=0; i < num_normalizers; i++)
     {
         if (i%2 != num_normalizers%2)
diff --git a/src/service_inspectors/http_inspect/http_header_normalizer.h b/src/service_inspectors/http_inspect/http_header_normalizer.h
index d0e3da51d..fad6a9182 100644
--- a/src/service_inspectors/http_inspect/http_header_normalizer.h
+++ b/src/service_inspectors/http_inspect/http_header_normalizer.h
@@ -38,8 +38,9 @@
 class HeaderNormalizer
 {
 public:
-    constexpr HeaderNormalizer(bool _concatenate_repeats, NormFunc* f1, NormFunc* f2, NormFunc* f3)
-        : concatenate_repeats(_concatenate_repeats), normalizer { f1, f2, f3 },
+    constexpr HeaderNormalizer(HttpEnums::EventSid _repeat_event,
+        HttpEnums::Infraction _repeat_inf, NormFunc* f1, NormFunc* f2, NormFunc* f3)
+        : repeat_event(_repeat_event), repeat_inf(_repeat_inf), normalizer { f1, f2, f3 },
         num_normalizers((f1 != nullptr) + (f1 != nullptr)*(f2 != nullptr) + (f1 != nullptr)*(f2 !=
             nullptr)*(f3 != nullptr)) { }
 
@@ -51,7 +52,8 @@ public:
 private:
     static int32_t derive_header_content(const uint8_t* value, int32_t length, uint8_t* buffer);
 
-    const bool concatenate_repeats;
+    const HttpEnums::EventSid repeat_event;
+    const HttpEnums::Infraction repeat_inf;
     NormFunc* const normalizer[3];
     const int num_normalizers;
 };
diff --git a/src/service_inspectors/http_inspect/http_infractions.h b/src/service_inspectors/http_inspect/http_infractions.h
index 67866bf73..deadc4580 100644
--- a/src/service_inspectors/http_inspect/http_infractions.h
+++ b/src/service_inspectors/http_inspect/http_infractions.h
@@ -33,7 +33,13 @@ class HttpInfractions
 {
 public:
     HttpInfractions() = default;
-    HttpInfractions(int inf) { assert((inf >= 0) && (inf < MAX)); infractions[inf] = true; }
+    HttpInfractions(int inf)
+    {
+        if (inf == HttpEnums::INF__NONE)
+            return;
+        assert((inf >= 0) && (inf < MAX));
+        infractions[inf] = true;
+    }
     bool none_found() const { return infractions == 0; }
     HttpInfractions& operator+=(const HttpInfractions& rhs)
         { infractions |= rhs.infractions; return *this; }
diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.cc b/src/service_inspectors/http_inspect/http_msg_head_shared.cc
index b60db3a58..8fa2f5187 100644
--- a/src/service_inspectors/http_inspect/http_msg_head_shared.cc
+++ b/src/service_inspectors/http_inspect/http_msg_head_shared.cc
@@ -171,9 +171,9 @@ void HttpMsgHeadShared::parse_header_lines()
     header_value = new Field[num_headers];
     header_name_id = new HeaderId[num_headers];
 
-    int colon;
     for (int k=0; k < num_headers; k++)
     {
+        int colon;
         for (colon=0; colon < header_line[k].length(); colon++)
         {
             if (header_line[k].start()[colon] == ':')
diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.h b/src/service_inspectors/http_inspect/http_msg_head_shared.h
index f65c0743e..fe49b6030 100644
--- a/src/service_inspectors/http_inspect/http_msg_head_shared.h
+++ b/src/service_inspectors/http_inspect/http_msg_head_shared.h
@@ -72,11 +72,15 @@ private:
     // Header normalization strategies. There should be one defined for every different way we can
     // process a header field value.
     static const HeaderNormalizer NORMALIZER_BASIC;
+    static const HeaderNormalizer NORMALIZER_NO_REPEAT;
+    static const HeaderNormalizer NORMALIZER_CASE_INSENSITIVE;
     static const HeaderNormalizer NORMALIZER_NUMBER;
     static const HeaderNormalizer NORMALIZER_TOKEN_LIST;
+    static const HeaderNormalizer NORMALIZER_METHOD_LIST;
+    static const HeaderNormalizer NORMALIZER_DATE;
+    static const HeaderNormalizer NORMALIZER_URI;
+    static const HeaderNormalizer NORMALIZER_CONTENT_LENGTH;
     static const HeaderNormalizer NORMALIZER_CHARSET;
-    static const HeaderNormalizer NORMALIZER_CAT;
-    static const HeaderNormalizer NORMALIZER_COOKIE;
 
     // Master table of known header fields and their normalization strategies.
     static const HeaderNormalizer* const header_norms[];
diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index b067f8bb4..50b779c8c 100644
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -57,11 +57,6 @@ void HttpMsgHeader::publish()
 
 void HttpMsgHeader::gen_events()
 {
-    if (get_header_count(HEAD_CONTENT_LENGTH) > 1)
-    {
-        add_infraction(INF_MULTIPLE_CONTLEN);
-        create_event(EVENT_MULTIPLE_CONTLEN);
-    }
     if ((get_header_count(HEAD_CONTENT_LENGTH) > 0) &&
         (get_header_count(HEAD_TRANSFER_ENCODING) > 0))
     {
@@ -165,6 +160,7 @@ void HttpMsgHeader::update_flow()
     // else because Transfer-Encoding header negates Content-Length header even if something was
     // wrong with Transfer-Encoding header. However a Transfer-Encoding header in a 1.0 message
     // does not negate the Content-Length header.
+    // FIXIT-L the following can be zero, need an alert for empty CL header value
     else if (get_header_value_norm(HEAD_CONTENT_LENGTH).length() > 0)
     {
         const int64_t content_length =
diff --git a/src/service_inspectors/http_inspect/http_normalizers.cc b/src/service_inspectors/http_inspect/http_normalizers.cc
index d6521b188..5ac6d389f 100644
--- a/src/service_inspectors/http_inspect/http_normalizers.cc
+++ b/src/service_inspectors/http_inspect/http_normalizers.cc
@@ -69,14 +69,16 @@ int32_t norm_remove_quotes_lws(const uint8_t* in_buf, int32_t in_length, uint8_t
 }
 
 // Other header-value processing functions (not using the standard normalization signature)
-// Convert a decimal field such as Content-Length to an integer.
+// Convert a decimal field such as Content-Length to an integer. If multiple comma-separated
+// values use the first one.
 int64_t norm_decimal_integer(const Field& input)
 {
     assert(input.length() > 0);
     // Limited to 18 decimal digits, not including leading zeros, to fit comfortably into int64_t
     int64_t total = 0;
     int non_leading_zeros = 0;
-    for (int32_t k=0; k < input.length(); k++)
+    int32_t k=0;
+    do
     {
         int value = input.start()[k] - '0';
         if ((non_leading_zeros > 0) || (value != 0))
@@ -87,6 +89,7 @@ int64_t norm_decimal_integer(const Field& input)
             return STAT_PROBLEMATIC;
         total = total*10 + value;
     }
+    while ((++k < input.length()) && (input.start()[k] != ','));
     return total;
 }
 
diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc
index dcb51f11f..9c5bde32c 100644
--- a/src/service_inspectors/http_inspect/http_tables.cc
+++ b/src/service_inspectors/http_inspect/http_tables.cc
@@ -134,6 +134,7 @@ const StrCode HttpMsgHeadShared::header_list[] =
     { HEAD_TRUE_CLIENT_IP,            "true-client-ip" },
     { HEAD_X_WORKING_WITH,            "x-working-with" },
     { HEAD_CONTENT_TRANSFER_ENCODING, "content-transfer-encoding" },
+    { HEAD_MIME_VERSION,              "mime-version" },
     { 0,                              nullptr }
 };
 
@@ -169,22 +170,39 @@ const StrCode HttpMsgHeadShared::charset_code_opt_list[] =
 };
 
 const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_BASIC
-    { false, nullptr, nullptr, nullptr };
+    { EVENT__NONE, INF__NONE, nullptr, nullptr, nullptr };
+
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_NO_REPEAT
+    { EVENT_REPEATED_HEADER, INF_REPEATED_HEADER, nullptr, nullptr, nullptr };
+
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CASE_INSENSITIVE
+    { EVENT__NONE, INF__NONE, norm_to_lower, nullptr, nullptr };
 
 const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_NUMBER
-    { false, norm_remove_lws, nullptr, nullptr };
+    { EVENT_REPEATED_HEADER, INF_REPEATED_HEADER, norm_remove_lws, nullptr, nullptr };
 
 const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_TOKEN_LIST
-    { true, norm_remove_lws, norm_to_lower, nullptr };
+    { EVENT__NONE, INF__NONE, norm_remove_lws, norm_to_lower, nullptr };
 
-const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CHARSET
-    { true, norm_remove_quotes_lws, norm_to_lower, nullptr };
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_METHOD_LIST
+    { EVENT__NONE, INF__NONE, norm_remove_lws, nullptr, nullptr };
 
-const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CAT
-    { true, norm_remove_lws, nullptr, nullptr };
+// FIXIT-L implement a date normalization function that converts the three legal formats into a
+// single standard format. For now we do nothing special for dates. This object is a placeholder
+// to keep track of which headers have date values.
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_DATE
+    { EVENT__NONE, INF__NONE, nullptr, nullptr, nullptr };
 
-const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_COOKIE
-    { true, nullptr, nullptr, nullptr };
+// FIXIT-M implement a URI normalization function, probably by extending existing URI capabilities
+// to cover relative formats
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_URI
+    { EVENT__NONE, INF__NONE, nullptr, nullptr, nullptr };
+
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CONTENT_LENGTH
+    { EVENT_MULTIPLE_CONTLEN, INF_MULTIPLE_CONTLEN, norm_remove_lws, nullptr, nullptr };
+
+const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CHARSET
+    { EVENT__NONE, INF__NONE, norm_remove_quotes_lws, norm_to_lower, nullptr };
 
 #if defined(__clang__)
 // Designated initializers are not supported in C++11. However we're going to play compilation
@@ -197,59 +215,60 @@ const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_COOKIE
 const HeaderNormalizer* const HttpMsgHeadShared::header_norms[HEAD__MAX_VALUE] = {
     [0] = &NORMALIZER_BASIC,
     [HEAD__OTHER] = &NORMALIZER_BASIC,
-    [HEAD_CACHE_CONTROL] = &NORMALIZER_BASIC,
-    [HEAD_CONNECTION] = &NORMALIZER_BASIC,
-    [HEAD_DATE] = &NORMALIZER_BASIC,
-    [HEAD_PRAGMA] = &NORMALIZER_BASIC,
-    [HEAD_TRAILER] = &NORMALIZER_BASIC,
-    [HEAD_COOKIE] = &NORMALIZER_COOKIE,
-    [HEAD_SET_COOKIE] = &NORMALIZER_COOKIE,
+    [HEAD_CACHE_CONTROL] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_CONNECTION] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_DATE] = &NORMALIZER_DATE,
+    [HEAD_PRAGMA] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_TRAILER] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_COOKIE] = &NORMALIZER_BASIC,
+    [HEAD_SET_COOKIE] = &NORMALIZER_BASIC,
     [HEAD_TRANSFER_ENCODING] = &NORMALIZER_TOKEN_LIST,
     [HEAD_UPGRADE] = &NORMALIZER_BASIC,
     [HEAD_VIA] = &NORMALIZER_BASIC,
     [HEAD_WARNING] = &NORMALIZER_BASIC,
-    [HEAD_ACCEPT] = &NORMALIZER_BASIC,
-    [HEAD_ACCEPT_CHARSET] = &NORMALIZER_BASIC,
-    [HEAD_ACCEPT_ENCODING] = &NORMALIZER_CAT,
-    [HEAD_ACCEPT_LANGUAGE] = &NORMALIZER_CAT,
+    [HEAD_ACCEPT] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_ACCEPT_CHARSET] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_ACCEPT_ENCODING] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_ACCEPT_LANGUAGE] = &NORMALIZER_TOKEN_LIST,
     [HEAD_AUTHORIZATION] = &NORMALIZER_BASIC,
-    [HEAD_EXPECT] = &NORMALIZER_BASIC,
+    [HEAD_EXPECT] = &NORMALIZER_CASE_INSENSITIVE,
     [HEAD_FROM] = &NORMALIZER_BASIC,
-    [HEAD_HOST] = &NORMALIZER_BASIC,
+    [HEAD_HOST] = &NORMALIZER_NO_REPEAT,
     [HEAD_IF_MATCH] = &NORMALIZER_BASIC,
-    [HEAD_IF_MODIFIED_SINCE] = &NORMALIZER_BASIC,
+    [HEAD_IF_MODIFIED_SINCE] = &NORMALIZER_DATE,
     [HEAD_IF_NONE_MATCH] = &NORMALIZER_BASIC,
     [HEAD_IF_RANGE] = &NORMALIZER_BASIC,
-    [HEAD_IF_UNMODIFIED_SINCE] = &NORMALIZER_BASIC,
+    [HEAD_IF_UNMODIFIED_SINCE] = &NORMALIZER_DATE,
     [HEAD_MAX_FORWARDS] = &NORMALIZER_BASIC,
     [HEAD_PROXY_AUTHORIZATION] = &NORMALIZER_BASIC,
     [HEAD_RANGE] = &NORMALIZER_BASIC,
-    [HEAD_REFERER] = &NORMALIZER_BASIC,
-    [HEAD_TE] = &NORMALIZER_BASIC,
+    [HEAD_REFERER] = &NORMALIZER_URI,
+    [HEAD_TE] = &NORMALIZER_TOKEN_LIST,
     [HEAD_USER_AGENT] = &NORMALIZER_BASIC,
-    [HEAD_ACCEPT_RANGES] = &NORMALIZER_BASIC,
-    [HEAD_AGE] = &NORMALIZER_BASIC,
+    [HEAD_ACCEPT_RANGES] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_AGE] = &NORMALIZER_NUMBER,
     [HEAD_ETAG] = &NORMALIZER_BASIC,
-    [HEAD_LOCATION] = &NORMALIZER_BASIC,
+    [HEAD_LOCATION] = &NORMALIZER_URI,
     [HEAD_PROXY_AUTHENTICATE] = &NORMALIZER_BASIC,
-    [HEAD_RETRY_AFTER] = &NORMALIZER_BASIC,
+    [HEAD_RETRY_AFTER] = &NORMALIZER_BASIC,  // may be date or number
     [HEAD_SERVER] = &NORMALIZER_BASIC,
-    [HEAD_VARY] = &NORMALIZER_BASIC,
+    [HEAD_VARY] = &NORMALIZER_TOKEN_LIST,
     [HEAD_WWW_AUTHENTICATE] = &NORMALIZER_BASIC,
-    [HEAD_ALLOW] = &NORMALIZER_BASIC,
+    [HEAD_ALLOW] = &NORMALIZER_METHOD_LIST,
     [HEAD_CONTENT_ENCODING] = &NORMALIZER_TOKEN_LIST,
-    [HEAD_CONTENT_LANGUAGE] = &NORMALIZER_BASIC,
-    [HEAD_CONTENT_LENGTH] = &NORMALIZER_NUMBER,
-    [HEAD_CONTENT_LOCATION] = &NORMALIZER_BASIC,
+    [HEAD_CONTENT_LANGUAGE] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_CONTENT_LENGTH] = &NORMALIZER_CONTENT_LENGTH,
+    [HEAD_CONTENT_LOCATION] = &NORMALIZER_URI,
     [HEAD_CONTENT_MD5] = &NORMALIZER_BASIC,
     [HEAD_CONTENT_RANGE] = &NORMALIZER_BASIC,
     [HEAD_CONTENT_TYPE] = &NORMALIZER_CHARSET,
-    [HEAD_EXPIRES] = &NORMALIZER_BASIC,
-    [HEAD_LAST_MODIFIED] = &NORMALIZER_BASIC,
-    [HEAD_X_FORWARDED_FOR] = &NORMALIZER_CAT,
+    [HEAD_EXPIRES] = &NORMALIZER_DATE,
+    [HEAD_LAST_MODIFIED] = &NORMALIZER_DATE,
+    [HEAD_X_FORWARDED_FOR] = &NORMALIZER_BASIC,
     [HEAD_TRUE_CLIENT_IP] = &NORMALIZER_BASIC,
     [HEAD_X_WORKING_WITH] = &NORMALIZER_BASIC,
-    [HEAD_CONTENT_TRANSFER_ENCODING] = &NORMALIZER_CAT,
+    [HEAD_CONTENT_TRANSFER_ENCODING] = &NORMALIZER_TOKEN_LIST,
+    [HEAD_MIME_VERSION] = &NORMALIZER_BASIC,
 };
 /* *INDENT-ON* */
 
@@ -353,6 +372,8 @@ const RuleMap HttpModule::http_events[] =
     { EVENT_CHUNKED_ONE_POINT_ZERO,     "HTTP 1.0 message with Transfer-Encoding header" },
     { EVENT_CTE_HEADER,                 "Content-Transfer-Encoding used as HTTP header" },
     { EVENT_ILLEGAL_TRAILER,            "illegal field in chunked message trailers" },
+    { EVENT_REPEATED_HEADER,            "header field inappropriately appears twice or has two "
+                                        "values" },
     { 0, nullptr }
 };
 
diff --git a/src/service_inspectors/http_inspect/test/CMakeLists.txt b/src/service_inspectors/http_inspect/test/CMakeLists.txt
index 8706c9dab..89de05c0f 100644
--- a/src/service_inspectors/http_inspect/test/CMakeLists.txt
+++ b/src/service_inspectors/http_inspect/test/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_cpputest(http_uri_norm_test http_inspect framework)
+add_cpputest(http_normalizers_test http_inspect framework)
 add_cpputest(http_module_test http_inspect framework)
 add_cpputest(http_msg_head_shared_util_test http_inspect framework)
 
diff --git a/src/service_inspectors/http_inspect/test/Makefile.am b/src/service_inspectors/http_inspect/test/Makefile.am
index da3b66472..41ce9b0e3 100644
--- a/src/service_inspectors/http_inspect/test/Makefile.am
+++ b/src/service_inspectors/http_inspect/test/Makefile.am
@@ -3,6 +3,7 @@ AM_DEFAULT_SOURCE_EXT = .cc
 
 check_PROGRAMS = \
 http_uri_norm_test \
+http_normalizers_test \
 http_module_test \
 http_transaction_test \
 http_msg_head_shared_util_test
@@ -22,6 +23,12 @@ http_uri_norm_test_LDADD = \
 ../../../framework/module.o \
 @CPPUTEST_LDFLAGS@
 
+http_normalizers_test_CPPFLAGS = $(AM_CPPFLAGS) @CPPUTEST_CPPFLAGS@
+http_normalizers_test_LDADD = \
+../http_normalizers.o \
+../http_field.o \
+@CPPUTEST_LDFLAGS@
+
 http_module_test_CPPFLAGS = $(AM_CPPFLAGS) @CPPUTEST_CPPFLAGS@
 http_module_test_LDADD = \
 ../http_module.o \
diff --git a/src/service_inspectors/http_inspect/test/http_normalizers_test.cc b/src/service_inspectors/http_inspect/test/http_normalizers_test.cc
new file mode 100644
index 000000000..4c100c84a
--- /dev/null
+++ b/src/service_inspectors/http_inspect/test/http_normalizers_test.cc
@@ -0,0 +1,71 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2016-2017 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// http_normalizers_test.cc author Tom Peters <thopeter@cisco.com>
+// unit test main
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "service_inspectors/http_inspect/http_field.h"
+#include "service_inspectors/http_inspect/http_normalizers.h"
+#include "service_inspectors/http_inspect/http_test_manager.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+#include <CppUTestExt/MockSupport.h>
+
+using namespace HttpEnums;
+
+// Stubs whose sole purpose is to make the test code link
+const bool HttpEnums::is_sp_tab[256] {};
+const bool HttpEnums::is_sp_tab_quote_dquote[256] {};
+long HttpTestManager::print_amount {};
+bool HttpTestManager::print_hex {};
+
+TEST_GROUP(norm_decimal_integer_test) {};
+
+TEST(norm_decimal_integer_test, examples)
+{
+    CHECK(norm_decimal_integer(Field(1, (const uint8_t*)"0")) == 0);
+    CHECK(norm_decimal_integer(Field(2, (const uint8_t*)"27")) == 27);
+    CHECK(norm_decimal_integer(Field(5, (const uint8_t*)"00027")) == 27);
+    CHECK(norm_decimal_integer(Field(2, (const uint8_t*)"-27")) == STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(6, (const uint8_t*)"27,382")) == 27);
+    CHECK(norm_decimal_integer(Field(3, (const uint8_t*)",27")) == STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(3, (const uint8_t*)",27")) == STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(6, (const uint8_t*)"00000=")) == STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(6, (const uint8_t*)"32.578")) == STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(18, (const uint8_t*)"123456789012345678")) ==
+        123456789012345678);
+    CHECK(norm_decimal_integer(Field(19, (const uint8_t*)"1234567890123456789")) ==
+        STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(19, (const uint8_t*)"0123456789012345678")) ==
+        123456789012345678);
+    CHECK(norm_decimal_integer(Field(20, (const uint8_t*)"01234567890123456789")) ==
+        STAT_PROBLEMATIC);
+    CHECK(norm_decimal_integer(Field(25, (const uint8_t*)"0000000000000000000000027")) == 27);
+    CHECK(norm_decimal_integer(Field(8, (const uint8_t*)"0040E,27")) == STAT_PROBLEMATIC);
+}
+
+int main(int argc, char** argv)
+{
+    return CommandLineTestRunner::RunAllTests(argc, argv);
+}
+