Merge pull request #353 in SNORT/snort3 from nhttp40 to master

Squashed commit of the following:

commit 6ed4f969db5564be8ab6d19f1b47ec809a42805f
Author: Tom Peters <thopeter@cisco.com>
Date:   Mon Mar 21 12:44:44 2016 -0400

    error to warning change

commit 3c8100967e88f792dd350e9c42a88ba2ef364aea
Author: Tom Peters <thopeter@cisco.com>
Date:   Wed Mar 16 15:38:50 2016 -0400

    UTF-8 Bare Byte

diff --git a/src/service_inspectors/nhttp_inspect/ips_nhttp.cc b/src/service_inspectors/nhttp_inspect/ips_nhttp.cc
index 7b7934b22c68be35660fd539c06bf3dc628925cb..c171366b37283374b0fd3df7b5036cae1fdd3321 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/ips_nhttp.cc
+++ b/src/service_inspectors/nhttp_inspect/ips_nhttp.cc
@@ -24,6 +24,7 @@
 #include "detection/detection_defines.h"
 #include "framework/cursor.h"
 #include "hash/sfhashfcn.h"
+#include "log/messages.h"
 
 #include "nhttp_inspect.h"
 #include "nhttp_msg_head_shared.h"

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_enum.h b/src/service_inspectors/nhttp_inspect/nhttp_enum.h
index 61cce6360465f4e352370f1501856a12c9086f8a..c4dcffc393c997cf1fdcc5445cbf32fcf59fda8e 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_enum.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_enum.h
@@ -112,7 +112,7 @@ enum HeaderId { HEAD__NOT_COMPUTE=-14, HEAD__PROBLEMATIC=-12, HEAD__NOT_PRESENT=
 // All the infractions we might find while parsing and analyzing a message
 enum Infraction
 {
-    INF_NOT_USED_1 = 0,
+    INF_BARE_BYTE = 0,
     INF_HEAD_TOO_LONG,
     INF_BAD_REQ_LINE,
     INF_BAD_STAT_LINE,

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
index dc33be32ead959a68e26cf1276abcb736d141429..27d55119909279a1b269eb93d77829fe6244cf4a 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
@@ -132,32 +132,32 @@ const Field& NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Fl
     switch (session_data->section_type[source_id])
     {
     case SEC_REQUEST:
-        latest_section = new NHttpMsgRequest(data, dsize, session_data, source_id, buf_owner,
-            flow, params);
+        latest_section = new NHttpMsgRequest(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_STATUS:
-        latest_section = new NHttpMsgStatus(data, dsize, session_data, source_id, buf_owner, flow,
-            params);
+        latest_section = new NHttpMsgStatus(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_HEADER:
-        latest_section = new NHttpMsgHeader(data, dsize, session_data, source_id, buf_owner, flow,
-            params);
+        latest_section = new NHttpMsgHeader(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_BODY_CL:
-        latest_section = new NHttpMsgBodyCl(data, dsize, session_data, source_id, buf_owner, flow,
-            params);
+        latest_section = new NHttpMsgBodyCl(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_BODY_OLD:
-        latest_section = new NHttpMsgBodyOld(data, dsize, session_data, source_id, buf_owner, flow,
-            params);
+        latest_section = new NHttpMsgBodyOld(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_BODY_CHUNK:
-        latest_section = new NHttpMsgBodyChunk(data, dsize, session_data, source_id, buf_owner,
-            flow, params);
+        latest_section = new NHttpMsgBodyChunk(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     case SEC_TRAILER:
-        latest_section = new NHttpMsgTrailer(data, dsize, session_data, source_id, buf_owner,
-            flow, params);
+        latest_section = new NHttpMsgTrailer(
+            data, dsize, session_data, source_id, buf_owner, flow, params);
         break;
     default:
         assert(false);

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_module.cc b/src/service_inspectors/nhttp_inspect/nhttp_module.cc
index 2d43317dbd9c31d63e061a6b8b58e6116197fbba..5d20b54a63ad8115886ee79cb252c2a2e1590742 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_module.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_module.cc
@@ -20,6 +20,8 @@
 #include <string.h>
 #include <sys/types.h>
 
+#include "log/messages.h"
+
 #include "nhttp_uri_norm.h"
 #include "nhttp_module.h"
 
@@ -37,8 +39,11 @@ const Parameter NHttpModule::nhttp_params[] =
     { "ignore_unreserved", Parameter::PT_STRING, "(optional)", nullptr,
           "do not alert when the specified unreserved characters are percent-encoded in a URI."
           "Unreserved characters are 0-9, a-z, A-Z, period, underscore, tilde, and minus." },
+    { "percent_u", Parameter::PT_BOOL, nullptr, "false", "normalize %uNNNN and %UNNNN encodings" },
     { "utf8", Parameter::PT_BOOL, nullptr, "true",
           "normalize 2-byte and 3-byte UTF-8 characters to a single byte" },
+    { "utf8_bare_byte", Parameter::PT_BOOL, nullptr, "false",
+          "when doing UTF-8 character normalization include bytes that were not percent encoded" },
     { "iis_unicode", Parameter::PT_BOOL, nullptr, "false",
           "use IIS unicode codepoint mapping to normalize characters" },
     { "backslash_to_slash", Parameter::PT_BOOL, nullptr, "false",
@@ -91,10 +96,18 @@ bool NHttpModule::set(const char*, Value& val, SnortConfig*)
             params->uri_param.unreserved_char[*(ignore++)] = false;
         }
     }
+    else if (val.is("percent_u"))
+    {
+        params->uri_param.percent_u = val.get_bool();
+    }
     else if (val.is("utf8"))
     {
         params->uri_param.utf8 = val.get_bool();
     }
+    else if (val.is("utf8_bare_byte"))
+    {
+        params->uri_param.utf8_bare_byte = val.get_bool();
+    }
     else if (val.is("iis_unicode"))
     {
         params->uri_param.iis_unicode = val.get_bool();
@@ -145,6 +158,16 @@ bool NHttpModule::set(const char*, Value& val, SnortConfig*)
     return true;
 }
 
+bool NHttpModule::end(const char*, int, SnortConfig*)
+{
+    if (!params->uri_param.utf8 && params->uri_param.utf8_bare_byte)
+    {
+        ParseWarning(WARN_CONF, "Meaningless to do bare byte when not doing UTF-8");
+        params->uri_param.utf8_bare_byte = false;
+    }
+    return true;
+}
+
 // Some values in these tables may be changed by configuration parameters.
 NHttpParaList::UriParam::UriParam() :
   // Characters that should not be percent-encoded

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_module.h b/src/service_inspectors/nhttp_inspect/nhttp_module.h
index 5f9af28455a8797c4c65c0290e8afb8b791a7444..fe2c10590623a3a706eefb86694a73aba22851e1 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_module.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_module.h
@@ -42,7 +42,9 @@ public:
         UriParam();
         ~UriParam() { delete[] unicode_map; }
 
+        bool percent_u;
         bool utf8;
+        bool utf8_bare_byte;
         bool iis_unicode;
         uint8_t* unicode_map = nullptr;
         bool backslash_to_slash;
@@ -67,7 +69,7 @@ public:
     NHttpModule() : Module(NHTTP_NAME, NHTTP_HELP, nhttp_params) { }
     ~NHttpModule() { delete params; }
     bool begin(const char*, int, SnortConfig*) override;
-    bool end(const char*, int, SnortConfig*) override { return true; }
+    bool end(const char*, int, SnortConfig*) override;
     bool set(const char*, Value&, SnortConfig*) override;
     unsigned get_gid() const override { return NHttpEnums::NHTTP_GID; }
     const RuleMap* get_rules() const override { return nhttp_events; }

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
index 4f76150e974cfa5fe5bce77e0ad5ae5e44811545..e2dc5f85c84e72ab1ce962a0d7b579d0465ca265 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
@@ -139,9 +139,13 @@ int32_t UriNormalizer::norm_percent_processing(const Field& input, uint8_t* out_
     {
         switch (uri_param.uri_char[input.start[k]])
         {
+        case CHAR_EIGHTBIT:
+            if (uri_param.utf8_bare_byte &&
+               (((input.start[k] & 0xE0) == 0xC0) || ((input.start[k] & 0xF0) == 0xE0)))
+                utf8_needed = true;
+            // Fall through
         case CHAR_NORMAL:
         case CHAR_PATH:
-        case CHAR_EIGHTBIT:
         case CHAR_SUBSTIT:
             out_buf[length++] = input.start[k];
             break;
@@ -152,7 +156,7 @@ int32_t UriNormalizer::norm_percent_processing(const Field& input, uint8_t* out_
                 // %hh => hex value
                 const uint8_t hex_val = as_hex[input.start[k+1]] * 16 + as_hex[input.start[k+2]];
                 percent_encoded[length] = true;
-                // Test for start of two-byte (110xxxxx) or three-byte (1110xxxx) UTF-8
+                // Test for possible start of two-byte (110xxxxx) or three-byte (1110xxxx) UTF-8
                 if (((hex_val & 0xE0) == 0xC0) || ((hex_val & 0xF0) == 0xE0))
                     utf8_needed = true;
                 out_buf[length++] = hex_val;
@@ -204,16 +208,21 @@ int32_t UriNormalizer::norm_utf8_processing(const Field& input, uint8_t* out_buf
     int32_t length = 0;
     for (int32_t k=0; k < input.length; k++)
     {
-        if (percent_encoded[k])
+        if (percent_encoded[k] || uri_param.utf8_bare_byte)
         {
             // two-byte UTF-8: 110xxxxx 10xxxxxx
             if (((input.start[k] & 0xE0) == 0xC0) &&
                 (k+1 < input.length) &&
-                percent_encoded[k+1] &&
+                (percent_encoded[k+1] || uri_param.utf8_bare_byte) &&
                 ((input.start[k+1] & 0xC0) == 0x80))
             {
                 infractions += INF_URI_PERCENT_UTF8_2B;
                 events.create_event(EVENT_UTF_8);
+                if (!percent_encoded[k] || !percent_encoded[k+1])
+                {
+                    infractions += INF_BARE_BYTE;
+                    events.create_event(EVENT_BARE_BYTE);
+                }
                 const uint16_t utf8_val = ((input.start[k] & 0x1F) << 6) +
                                            (input.start[k+1] & 0x3F);
                 out_buf[length++] = reduce_to_eight_bits(utf8_val, uri_param, infractions, events);
@@ -222,13 +231,18 @@ int32_t UriNormalizer::norm_utf8_processing(const Field& input, uint8_t* out_buf
             // three-byte UTF-8: 1110xxxx 10xxxxxx 10xxxxxx
             else if (((input.start[k] & 0xF0) == 0xE0) &&
                 (k+2 < input.length) &&
-                percent_encoded[k+1] &&
+                (percent_encoded[k+1] || uri_param.utf8_bare_byte) &&
                 ((input.start[k+1] & 0xC0) == 0x80) &&
-                percent_encoded[k+2] &&
+                (percent_encoded[k+2] || uri_param.utf8_bare_byte) &&
                 ((input.start[k+2] & 0xC0) == 0x80))
             {
                 infractions += INF_URI_PERCENT_UTF8_3B;
                 events.create_event(EVENT_UTF_8);
+                if (!percent_encoded[k] || !percent_encoded[k+1] || !percent_encoded[k+2])
+                {
+                    infractions += INF_BARE_BYTE;
+                    events.create_event(EVENT_BARE_BYTE);
+                }
                 const uint16_t utf8_val = ((input.start[k] & 0x0F) << 12) +
                                           ((input.start[k+1] & 0x3F) << 6) +
                                            (input.start[k+2] & 0x3F);