Merge pull request #345 in SNORT/snort3 from nhttp39 to master

Squashed commit of the following:

commit a5f7f411c02e9a834e462485a6b3a17ed7cf0e22
Author: Tom Peters <thopeter@cisco.com>
Date:   Mon Mar 7 11:22:22 2016 -0500

    UTF-8 normalization for NHI

diff --git a/src/service_inspectors/http_inspect/hi_norm.cc b/src/service_inspectors/http_inspect/hi_norm.cc
index 2fa58c3ce59e398a4264055411ec21885d1fd597..1f74fc63d9d31149d795c19b846a2840ce435ef3 100644 (file)
--- a/src/service_inspectors/http_inspect/hi_norm.cc
+++ b/src/service_inspectors/http_inspect/hi_norm.cc
@@ -578,7 +578,7 @@ static int UTF8Decode(HI_SESSION* session, const u_char* start,
     {
         if (ServerConf->iis_unicode.on)
         {
-            // FIXIT-L iNorm is an int; is it guaranteed to be < 64K?
+            // iNorm is based on max 3-byte UTF-8 and hence always fits in 16 bits
             iNorm = ServerConf->iis_unicode_map[iNorm];
 
             if (iNorm == HI_UI_NON_ASCII_CODEPOINT)

diff --git a/src/service_inspectors/http_inspect/hi_ui_iis_unicode_map.cc b/src/service_inspectors/http_inspect/hi_ui_iis_unicode_map.cc
index 3e1ee90e1d5855a9cb6c9c6e4d39659435133d3d..46ade42299024cddb9c165385479360786fb75eb 100644 (file)
--- a/src/service_inspectors/http_inspect/hi_ui_iis_unicode_map.cc
+++ b/src/service_inspectors/http_inspect/hi_ui_iis_unicode_map.cc
@@ -281,6 +281,9 @@ int hi_ui_parse_iis_unicode_map(uint8_t** iis_unicode_map, char* filename,
 bool get_default_unicode_map(uint8_t*& map, int& page)
 {
     page = default_unicode_page;
+    // FIXIT-M This certainly looks wrong. Why isn't the background value for this table
+    // initialized to HI_UI_NON_ASCII_CODEPOINT instead of zero? Compare with
+    // hi_ui_parse_iis_unicode_map() above.
     map = (uint8_t*)SnortAlloc(65536*sizeof(uint8_t));
 
     std::stringstream ss(default_unicode_map);

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_api.cc b/src/service_inspectors/nhttp_inspect/nhttp_api.cc
index dbe033c71cba76edfcd09bc31d80418ef891e05f..a58eb0d02ed14e166b399f71eb4e94bc7c395553 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_api.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_api.cc
@@ -29,8 +29,8 @@ const char* NHttpApi::nhttp_help = "the new HTTP inspector!";
 
 Inspector* NHttpApi::nhttp_ctor(Module* mod)
 {
-    const NHttpModule* const nhttp_mod = (NHttpModule*)mod;
-    return new NHttpInspect(nhttp_mod->get_params());
+    NHttpModule* const nhttp_mod = (NHttpModule*)mod;
+    return new NHttpInspect(nhttp_mod->get_once_params());
 }
 
 const char* NHttpApi::classic_buffers[] =

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_enum.h b/src/service_inspectors/nhttp_inspect/nhttp_enum.h
index 740c74cae008a65555b541cffe5b17074536e471..61cce6360465f4e352370f1501856a12c9086f8a 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_enum.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_enum.h
@@ -122,7 +122,7 @@ enum Infraction
     INF_UNKNOWN_VERSION,
     INF_BAD_VERSION,
     INF_ZERO_NINE_NOT_FIRST,
-    INF_NOT_USED_5,
+    INF_URI_IIS_UNICODE,
     INF_BAD_HEADER_DATA,
     INF_PIPELINE_OVERFLOW,
     INF_BAD_CHUNK_SIZE,
@@ -130,9 +130,9 @@ enum Infraction
     INF_BAD_URI,
     INF_ZERO_NINE_REQ,
     INF_ZERO_NINE_CONTINUE,
-    INF_NOT_USED_2,
+    INF_URI_PERCENT_UTF8_3B,
     INF_URI_PERCENT_UNRESERVED,
-    INF_URI_PERCENT_UTF8,
+    INF_URI_PERCENT_UTF8_2B,
     INF_URI_PERCENT_UCODE,
     INF_URI_PERCENT_OTHER,
     INF_URI_BAD_CHAR,

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_field.cc b/src/service_inspectors/nhttp_inspect/nhttp_field.cc
index e00ad0fb966200e327b2dc94331673a594e1919c..6ea3cdcc76ef9166a40b10e232026684f817654d 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_field.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_field.cc
@@ -73,7 +73,7 @@ void Field::print(FILE* output, const char* name) const
     // Limit the amount of data printed
     const int32_t print_length = (length <= NHttpTestManager::get_print_amount()) ? length :
         NHttpTestManager::get_print_amount();
-    for (int k=0; k < print_length; k++)
+    for (int32_t k=0; k < print_length; k++)
     {
         if ((start[k] >= 0x20) && (start[k] <= 0x7E))
             fprintf(output, "%c", (char)start[k]);
@@ -81,6 +81,8 @@ void Field::print(FILE* output, const char* name) const
             fprintf(output, "~");
         else if (start[k] == 0xA)
             fprintf(output, "^");
+        else if (NHttpTestManager::get_print_hex())
+            fprintf(output, "[%.2x]", (uint8_t)start[k]);
         else
             fprintf(output, "*");
         if ((k%120 == (119 - out_count)) && (k+1 < print_length))

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
index d20cee2a2b0fdeac0e12e5cd7db292a0fe711f0f..dc33be32ead959a68e26cf1276abcb736d141429 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc
@@ -39,18 +39,19 @@
 
 using namespace NHttpEnums;
 
-NHttpInspect::NHttpInspect(NHttpParaList params_) : params(params_)
+NHttpInspect::NHttpInspect(const NHttpParaList* params_) : params(params_)
 {
 #ifdef REG_TEST
-    if (params.test_input)
+    if (params->test_input)
     {
         NHttpTestManager::activate_test_input();
     }
-    if (params.test_output)
+    if (params->test_output)
     {
         NHttpTestManager::activate_test_output();
     }
-    NHttpTestManager::set_print_amount(params.print_amount);
+    NHttpTestManager::set_print_amount(params->print_amount);
+    NHttpTestManager::set_print_hex(params->print_hex);
 #endif
 }
 
@@ -132,31 +133,31 @@ const Field& NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Fl
     {
     case SEC_REQUEST:
         latest_section = new NHttpMsgRequest(data, dsize, session_data, source_id, buf_owner,
-            flow, &params);
+            flow, params);
         break;
     case SEC_STATUS:
         latest_section = new NHttpMsgStatus(data, dsize, session_data, source_id, buf_owner, flow,
-            &params);
+            params);
         break;
     case SEC_HEADER:
         latest_section = new NHttpMsgHeader(data, dsize, session_data, source_id, buf_owner, flow,
-            &params);
+            params);
         break;
     case SEC_BODY_CL:
         latest_section = new NHttpMsgBodyCl(data, dsize, session_data, source_id, buf_owner, flow,
-            &params);
+            params);
         break;
     case SEC_BODY_OLD:
         latest_section = new NHttpMsgBodyOld(data, dsize, session_data, source_id, buf_owner, flow,
-            &params);
+            params);
         break;
     case SEC_BODY_CHUNK:
         latest_section = new NHttpMsgBodyChunk(data, dsize, session_data, source_id, buf_owner,
-            flow, &params);
+            flow, params);
         break;
     case SEC_TRAILER:
         latest_section = new NHttpMsgTrailer(data, dsize, session_data, source_id, buf_owner,
-            flow, &params);
+            flow, params);
         break;
     default:
         assert(false);

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h
index ec003953112bd17f4755fd7c11c350adbe142dc1..07ef01f0f021a20da84de6d9699b35feece4bcbd 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h
@@ -39,7 +39,8 @@ class NHttpInspect : public Inspector
 public:
     static THREAD_LOCAL uint8_t body_buffer[NHttpEnums::MAX_OCTETS];
 
-    NHttpInspect(NHttpParaList params_);
+    NHttpInspect(const NHttpParaList* params_);
+    ~NHttpInspect() { delete params; }
 
     bool get_buf(InspectionBuffer::Type ibt, Packet*, InspectionBuffer& b) override;
     bool nhttp_get_buf(unsigned id, uint64_t sub_id, uint64_t form, Packet*, InspectionBuffer& b);
@@ -68,7 +69,7 @@ private:
 
     static THREAD_LOCAL NHttpMsgSection* latest_section;
 
-    const NHttpParaList params;
+    const NHttpParaList* const params;
 };
 
 #endif

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_module.cc b/src/service_inspectors/nhttp_inspect/nhttp_module.cc
index fde57baad469075575ba5dff51a4a472bbdc7a85..2d43317dbd9c31d63e061a6b8b58e6116197fbba 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_module.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_module.cc
@@ -37,6 +37,10 @@ const Parameter NHttpModule::nhttp_params[] =
     { "ignore_unreserved", Parameter::PT_STRING, "(optional)", nullptr,
           "do not alert when the specified unreserved characters are percent-encoded in a URI."
           "Unreserved characters are 0-9, a-z, A-Z, period, underscore, tilde, and minus." },
+    { "utf8", Parameter::PT_BOOL, nullptr, "true",
+          "normalize 2-byte and 3-byte UTF-8 characters to a single byte" },
+    { "iis_unicode", Parameter::PT_BOOL, nullptr, "false",
+          "use IIS unicode codepoint mapping to normalize characters" },
     { "backslash_to_slash", Parameter::PT_BOOL, nullptr, "false",
           "replace \\ with / when normalizing URIs" },
     { "plus_to_space", Parameter::PT_BOOL, nullptr, "true",
@@ -48,12 +52,16 @@ const Parameter NHttpModule::nhttp_params[] =
     { "test_output", Parameter::PT_BOOL, nullptr, "false", "print out HTTP section data" },
     { "print_amount", Parameter::PT_INT, "1:1000000", "1200",
           "number of characters to print from a Field" },
+    { "print_hex", Parameter::PT_BOOL, nullptr, "false",
+      "nonprinting characters printed in [HH] format instead of using an asterisk" },
 #endif
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
 bool NHttpModule::begin(const char*, int, SnortConfig*)
 {
+    delete params;
+    params = new NHttpParaList;
     return true;
 }
 
@@ -61,56 +69,73 @@ bool NHttpModule::set(const char*, Value& val, SnortConfig*)
 {
     if (val.is("request_depth"))
     {
-        params.request_depth = val.get_long();
+        params->request_depth = val.get_long();
     }
     else if (val.is("response_depth"))
     {
-        params.response_depth = val.get_long();
+        params->response_depth = val.get_long();
     }
     else if (val.is("unzip"))
     {
-        params.unzip = val.get_bool();
+        params->unzip = val.get_bool();
     }
     else if (val.is("bad_characters"))
     {
-        val.get_bits(params.uri_param.bad_characters);
+        val.get_bits(params->uri_param.bad_characters);
     }
     else if (val.is("ignore_unreserved"))
     {
         const char* ignore = val.get_string();
         while (*ignore != '\0')
         {
-            params.uri_param.unreserved_char[*(ignore++)] = false;
+            params->uri_param.unreserved_char[*(ignore++)] = false;
+        }
+    }
+    else if (val.is("utf8"))
+    {
+        params->uri_param.utf8 = val.get_bool();
+    }
+    else if (val.is("iis_unicode"))
+    {
+        params->uri_param.iis_unicode = val.get_bool();
+        if (params->uri_param.iis_unicode)
+        {
+            params->uri_param.unicode_map = new uint8_t[65536];
+            UriNormalizer::load_default_unicode_map(params->uri_param.unicode_map);
         }
     }
     else if (val.is("backslash_to_slash"))
     {
-        params.uri_param.backslash_to_slash = val.get_bool();
-        params.uri_param.uri_char[(uint8_t)'\\'] = val.get_bool() ? CHAR_SUBSTIT : CHAR_NORMAL;
+        params->uri_param.backslash_to_slash = val.get_bool();
+        params->uri_param.uri_char[(uint8_t)'\\'] = val.get_bool() ? CHAR_SUBSTIT : CHAR_NORMAL;
     }
     else if (val.is("plus_to_space"))
     {
-        params.uri_param.plus_to_space = val.get_bool();
-        params.uri_param.uri_char[(uint8_t)'+'] = val.get_bool() ? CHAR_SUBSTIT : CHAR_NORMAL;
+        params->uri_param.plus_to_space = val.get_bool();
+        params->uri_param.uri_char[(uint8_t)'+'] = val.get_bool() ? CHAR_SUBSTIT : CHAR_NORMAL;
     }
     else if (val.is("simplify_path"))
     {
-        params.uri_param.simplify_path = val.get_bool();
-        params.uri_param.uri_char[(uint8_t)'/'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL;
-        params.uri_param.uri_char[(uint8_t)'.'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL;
+        params->uri_param.simplify_path = val.get_bool();
+        params->uri_param.uri_char[(uint8_t)'/'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL;
+        params->uri_param.uri_char[(uint8_t)'.'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL;
     }
 #ifdef REG_TEST
     else if (val.is("test_input"))
     {
-        params.test_input = val.get_bool();
+        params->test_input = val.get_bool();
     }
     else if (val.is("test_output"))
     {
-        params.test_output = val.get_bool();
+        params->test_output = val.get_bool();
     }
     else if (val.is("print_amount"))
     {
-        params.print_amount = val.get_long();
+        params->print_amount = val.get_long();
+    }
+    else if (val.is("print_hex"))
+    {
+        params->print_hex = val.get_bool();
     }
 #endif
     else
@@ -150,7 +175,7 @@ NHttpParaList::UriParam::UriParam() :
     CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
     CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
     CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
-    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_PATH,      CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
+    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
 
     CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,
     CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,    CHAR_NORMAL,

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_module.h b/src/service_inspectors/nhttp_inspect/nhttp_module.h
index d4c3fd10ae5660e9ed0d4c08756c240feaccb78e..5f9af28455a8797c4c65c0290e8afb8b791a7444 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_module.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_module.h
@@ -40,7 +40,11 @@ public:
     {
     public:
         UriParam();
+        ~UriParam() { delete[] unicode_map; }
 
+        bool utf8;
+        bool iis_unicode;
+        uint8_t* unicode_map = nullptr;
         bool backslash_to_slash;
         bool plus_to_space;
         bool simplify_path;
@@ -53,6 +57,7 @@ public:
     bool test_input;
     bool test_output;
     long print_amount;
+    bool print_hex;
 #endif
 };
 
@@ -60,17 +65,23 @@ class NHttpModule : public Module
 {
 public:
     NHttpModule() : Module(NHTTP_NAME, NHTTP_HELP, nhttp_params) { }
+    ~NHttpModule() { delete params; }
     bool begin(const char*, int, SnortConfig*) override;
     bool end(const char*, int, SnortConfig*) override { return true; }
     bool set(const char*, Value&, SnortConfig*) override;
     unsigned get_gid() const override { return NHttpEnums::NHTTP_GID; }
     const RuleMap* get_rules() const override { return nhttp_events; }
-    NHttpParaList get_params() const { return params; }
+    const NHttpParaList* get_once_params()
+    {
+        NHttpParaList* ret_val = params;
+        params = nullptr;
+        return ret_val;
+    }
 
 private:
     static const Parameter nhttp_params[];
     static const RuleMap nhttp_events[];
-    NHttpParaList params;
+    NHttpParaList* params = nullptr;
 };
 
 #endif

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc
index 38b33e59e47133d3b4db433e5bcae5ff41645a13..52a4f6b6aa654a34c3695e15d6c1192197844211 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc
@@ -31,6 +31,7 @@ const char* NHttpTestManager::test_output_prefix = "nhttpresults/testcase";
 int64_t NHttpTestManager::test_number = -1;
 FILE* NHttpTestManager::test_out = nullptr;
 long NHttpTestManager::print_amount = 1200;
+bool NHttpTestManager::print_hex = false;
 
 void NHttpTestManager::update_test_number(int64_t new_test_number)
 {

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h
index 3e7829182a0a1b74aa90812326709354510d92bb..388032944a92fc8f020257495b98c0ed08f1ec85 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h
@@ -44,6 +44,8 @@ public:
     static int64_t get_test_number() { return test_number; }
     static void set_print_amount(long print_amount_) { print_amount = print_amount_; }
     static long get_print_amount() { return print_amount; }
+    static void set_print_hex(bool print_hex_) { print_hex = print_hex_; }
+    static bool get_print_hex() { return print_hex; }
 
 private:
     NHttpTestManager() = delete;
@@ -57,6 +59,7 @@ private:
     static FILE* test_out;
     static int64_t test_number;
     static long print_amount;
+    static bool print_hex;
 };
 
 #endif

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
index c34337f1ae267bdecde2de9c7d29c2a7c90c2185..4f76150e974cfa5fe5bce77e0ad5ae5e44811545 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
+++ b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.cc
@@ -20,6 +20,8 @@
 #include <assert.h>
 #include <sys/types.h>
 #include <cstring>
+#include <sstream>
+#include <string>
 
 #include "nhttp_enum.h"
 #include "nhttp_uri_norm.h"
@@ -100,7 +102,7 @@ bool UriNormalizer::need_norm_path(const Field& uri_component,
                     continue;
                 return true;
             }
-            else if (buf[k] == '.')
+            else
             {
                 // period is safe if not preceded or followed by another path character
                 if (((k == 0) || (uri_param.uri_char[buf[k-1]] != CHAR_PATH))          &&
@@ -108,10 +110,6 @@ bool UriNormalizer::need_norm_path(const Field& uri_component,
                     continue;
                 return true;
             }
-            else
-            {
-                return true;
-            }
         }
     }
     return false;
@@ -119,6 +117,22 @@ bool UriNormalizer::need_norm_path(const Field& uri_component,
 
 int32_t UriNormalizer::norm_char_clean(const Field& input, uint8_t* out_buf,
     const NHttpParaList::UriParam& uri_param, NHttpInfractions& infractions, NHttpEventGen& events)
+{
+    bool utf8_needed = false;
+    std::vector<bool> percent_encoded(input.length, false);
+    int32_t length = norm_percent_processing(input, out_buf, uri_param, utf8_needed,
+        percent_encoded, infractions, events);
+    if (uri_param.utf8 && utf8_needed)
+    {
+        length = norm_utf8_processing(Field(length, out_buf), out_buf, uri_param, percent_encoded,
+            infractions, events);
+    }
+    return length;
+}
+
+int32_t UriNormalizer::norm_percent_processing(const Field& input, uint8_t* out_buf,
+    const NHttpParaList::UriParam& uri_param, bool& utf8_needed,
+    std::vector<bool>& percent_encoded, NHttpInfractions& infractions, NHttpEventGen& events)
 {
     int32_t length = 0;
     for (int32_t k = 0; k < input.length; k++)
@@ -136,7 +150,12 @@ int32_t UriNormalizer::norm_char_clean(const Field& input, uint8_t* out_buf,
                 (as_hex[input.start[k+2]] != -1))
             {
                 // %hh => hex value
-                out_buf[length++] = as_hex[input.start[k+1]] * 16 + as_hex[input.start[k+2]];
+                const uint8_t hex_val = as_hex[input.start[k+1]] * 16 + as_hex[input.start[k+2]];
+                percent_encoded[length] = true;
+                // Test for start of two-byte (110xxxxx) or three-byte (1110xxxx) UTF-8
+                if (((hex_val & 0xE0) == 0xC0) || ((hex_val & 0xF0) == 0xE0))
+                    utf8_needed = true;
+                out_buf[length++] = hex_val;
                 k += 2;
             }
             else if ((k+1 < input.length) && (input.start[k+1] == '%'))
@@ -145,6 +164,20 @@ int32_t UriNormalizer::norm_char_clean(const Field& input, uint8_t* out_buf,
                 out_buf[length++] = '%';
                 k += 1;
             }
+            else if ((k+5 < input.length) &&
+                     ((input.start[k+1] == 'u') || (input.start[k+1] == 'U')) &&
+                     (as_hex[input.start[k+2]] != -1) &&
+                     (as_hex[input.start[k+3]] != -1) &&
+                     (as_hex[input.start[k+4]] != -1) &&
+                     (as_hex[input.start[k+5]] != -1) )
+            {
+                // %u encoding, this is nonstandard and likely to be malicious
+                out_buf[length++] = '\xFF';
+                k += 5;
+                // FIXIT-H this is a stub feature. Needs an event, a config option, computation,
+                // and unicode map lookup.
+                // Values > 0xFF become 0xFF unless they are in the code map
+            }
             else
             {
                 // don't recognize, pass through for now (FIXIT-H unfinished feature)
@@ -164,6 +197,68 @@ int32_t UriNormalizer::norm_char_clean(const Field& input, uint8_t* out_buf,
     return length;
 }
 
+int32_t UriNormalizer::norm_utf8_processing(const Field& input, uint8_t* out_buf,
+    const NHttpParaList::UriParam& uri_param, const std::vector<bool>& percent_encoded,
+    NHttpInfractions& infractions, NHttpEventGen& events)
+{
+    int32_t length = 0;
+    for (int32_t k=0; k < input.length; k++)
+    {
+        if (percent_encoded[k])
+        {
+            // two-byte UTF-8: 110xxxxx 10xxxxxx
+            if (((input.start[k] & 0xE0) == 0xC0) &&
+                (k+1 < input.length) &&
+                percent_encoded[k+1] &&
+                ((input.start[k+1] & 0xC0) == 0x80))
+            {
+                infractions += INF_URI_PERCENT_UTF8_2B;
+                events.create_event(EVENT_UTF_8);
+                const uint16_t utf8_val = ((input.start[k] & 0x1F) << 6) +
+                                           (input.start[k+1] & 0x3F);
+                out_buf[length++] = reduce_to_eight_bits(utf8_val, uri_param, infractions, events);
+                k += 1;
+            }
+            // three-byte UTF-8: 1110xxxx 10xxxxxx 10xxxxxx
+            else if (((input.start[k] & 0xF0) == 0xE0) &&
+                (k+2 < input.length) &&
+                percent_encoded[k+1] &&
+                ((input.start[k+1] & 0xC0) == 0x80) &&
+                percent_encoded[k+2] &&
+                ((input.start[k+2] & 0xC0) == 0x80))
+            {
+                infractions += INF_URI_PERCENT_UTF8_3B;
+                events.create_event(EVENT_UTF_8);
+                const uint16_t utf8_val = ((input.start[k] & 0x0F) << 12) +
+                                          ((input.start[k+1] & 0x3F) << 6) +
+                                           (input.start[k+2] & 0x3F);
+                out_buf[length++] = reduce_to_eight_bits(utf8_val, uri_param, infractions, events);
+                k += 2;
+            }
+            else
+                out_buf[length++] = input.start[k];
+        }
+        else
+            out_buf[length++] = input.start[k];
+    }
+    return length;
+}
+
+uint8_t UriNormalizer::reduce_to_eight_bits(uint16_t value,
+    const NHttpParaList::UriParam& uri_param, NHttpInfractions& infractions, NHttpEventGen& events)
+{
+    if (value <= 0xFF)
+        return value;
+    if (!uri_param.iis_unicode)
+        return 0xFF;
+    if (uri_param.unicode_map[value] != 0xFF)
+    {
+        infractions += INF_URI_IIS_UNICODE;
+        events.create_event(EVENT_IIS_UNICODE);
+    }
+    return uri_param.unicode_map[value];
+}
+
 void UriNormalizer::detect_bad_char(const Field& uri_component,
     const NHttpParaList::UriParam& uri_param, NHttpInfractions& infractions, NHttpEventGen& events)
 {
@@ -324,3 +419,53 @@ bool UriNormalizer::classic_need_norm(const Field& uri_component, bool do_path,
     return need_norm(uri_component, do_path, uri_param, unused, dummy_ev);
 }
 
+void UriNormalizer::load_default_unicode_map(uint8_t map[65536])
+{
+    memset(map, 0xFF, 65536);
+
+    // Default unicode map is just a single string of tokens of the form
+    // HHHH:HH (HHHH = unicode, HH = ascii char)
+    std::stringstream ss(
+  "0100:41 0101:61 0102:41 0103:61 0104:41 0105:61 0106:43 0107:63 0108:43 0109:63 010a:43 010b:63"
+  "010c:43 010d:63 010e:44 010f:64 0111:64 0112:45 0113:65 0114:45 0115:65 0116:45 0117:65 0118:45"
+  "0119:65 011a:45 011b:65 011c:47 011d:67 011e:47 011f:67 0120:47 0121:67 0122:47 0123:67 0124:48"
+  "0125:68 0126:48 0127:68 0128:49 0129:69 012a:49 012b:69 012c:49 012d:69 012e:49 012f:69 0130:49"
+  "0131:69 0134:4a 0135:6a 0136:4b 0137:6b 0139:4c 013a:6c 013b:4c 013c:6c 013d:4c 013e:6c 0141:4c"
+  "0142:6c 0143:4e 0144:6e 0145:4e 0146:6e 0147:4e 0148:6e 014c:4f 014d:6f 014e:4f 014f:6f 0150:4f"
+  "0151:6f 0154:52 0155:72 0156:52 0157:72 0158:52 0159:72 015a:53 015b:73 015c:53 015d:73 015e:53"
+  "015f:73 0162:54 0163:74 0164:54 0165:74 0166:54 0167:74 0168:55 0169:75 016a:55 016b:75 016c:55"
+  "016d:75 016e:55 016f:75 0170:55 0171:75 0172:55 0173:75 0174:57 0175:77 0176:59 0177:79 0179:5a"
+  "017b:5a 017c:7a 0180:62 0197:49 019a:6c 019f:4f 01a0:4f 01a1:6f 01ab:74 01ae:54 01af:55 01b0:75"
+  "01b6:7a 01c0:7c 01c3:21 01cd:41 01ce:61 01cf:49 01d0:69 01d1:4f 01d2:6f 01d3:55 01d4:75 01d5:55"
+  "01d6:75 01d7:55 01d8:75 01d9:55 01da:75 01db:55 01dc:75 01de:41 01df:61 01e4:47 01e5:67 01e6:47"
+  "01e7:67 01e8:4b 01e9:6b 01ea:4f 01eb:6f 01ec:4f 01ed:6f 01f0:6a 0261:67 02b9:27 02ba:22 02bc:27"
+  "02c4:5e 02c8:27 02cb:60 02cd:5f 0300:60 0302:5e 0303:7e 030e:22 0331:5f 0332:5f 037e:3b 0393:47"
+  "0398:54 03a3:53 03a6:46 03a9:4f 03b1:61 03b4:64 03b5:65 03c0:70 03c3:73 03c4:74 03c6:66 04bb:68"
+  "0589:3a 066a:25 2000:20 2001:20 2002:20 2003:20 2004:20 2005:20 2006:20 2010:2d 2011:2d 2017:3d"
+  "2032:27 2035:60 2044:2f 2074:34 2075:35 2076:36 2077:37 2078:38 207f:6e 2080:30 2081:31 2082:32"
+  "2083:33 2084:34 2085:35 2086:36 2087:37 2088:38 2089:39 20a7:50 2102:43 2107:45 210a:67 210b:48"
+  "210c:48 210d:48 210e:68 2110:49 2111:49 2112:4c 2113:6c 2115:4e 2118:50 2119:50 211a:51 211b:52"
+  "211c:52 211d:52 2124:5a 2128:5a 212a:4b 212c:42 212d:43 212e:65 212f:65 2130:45 2131:46 2133:4d"
+  "2134:6f 2212:2d 2215:2f 2216:5c 2217:2a 221a:76 221e:38 2223:7c 2229:6e 2236:3a 223c:7e 2261:3d"
+  "2264:3d 2265:3d 2303:5e 2320:28 2321:29 2329:3c 232a:3e 2500:2d 250c:2b 2510:2b 2514:2b 2518:2b"
+  "251c:2b 252c:2d 2534:2d 253c:2b 2550:2d 2552:2b 2553:2b 2554:2b 2555:2b 2556:2b 2557:2b 2558:2b"
+  "2559:2b 255a:2b 255b:2b 255c:2b 255d:2b 2564:2d 2565:2d 2566:2d 2567:2d 2568:2d 2569:2d 256a:2b"
+  "256b:2b 256c:2b 2584:5f 2758:7c 3000:20 3008:3c 3009:3e 301a:5b 301b:5d ff01:21 ff02:22 ff03:23"
+  "ff04:24 ff05:25 ff06:26 ff07:27 ff08:28 ff09:29 ff0a:2a ff0b:2b ff0c:2c ff0d:2d ff0e:2e ff0f:2f"
+  "ff10:30 ff11:31 ff12:32 ff13:33 ff14:34 ff15:35 ff16:36 ff17:37 ff18:38 ff19:39 ff1a:3a ff1b:3b"
+  "ff1c:3c ff1d:3d ff1e:3e ff20:40 ff21:41 ff22:42 ff23:43 ff24:44 ff25:45 ff26:46 ff27:47 ff28:48"
+  "ff29:49 ff2a:4a ff2b:4b ff2c:4c ff2d:4d ff2e:4e ff2f:4f ff30:50 ff31:51 ff32:52 ff33:53 ff34:54"
+  "ff35:55 ff36:56 ff37:57 ff38:58 ff39:59 ff3a:5a ff3b:5b ff3c:5c ff3d:5d ff3e:5e ff3f:5f ff40:60"
+  "ff41:61 ff42:62 ff43:63 ff44:64 ff45:65 ff46:66 ff47:67 ff48:68 ff49:69 ff4a:6a ff4b:6b ff4c:6c"
+  "ff4d:6d ff4e:6e ff4f:6f ff50:70 ff51:71 ff52:72 ff53:73 ff54:74 ff55:75 ff56:76 ff57:77 ff58:78"
+  "ff59:79 ff5a:7a ff5b:7b ff5c:7c ff5d:7d ff5e:7e");
+
+    std::string token;
+
+    while (ss >> token)
+    {
+        const uint16_t ucode = strtol(token.c_str(), nullptr, 16);
+        map[ucode] = strtol(token.c_str()+5, nullptr, 16);
+    }
+}
+

diff --git a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.h b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.h
index 6a3ba488896d12e0c58ed4e54baec8fe0631bb10..09a38e97dfb69c0b24bfe2cd0ee78235352e6eab 100644 (file)
--- a/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.h
+++ b/src/service_inspectors/nhttp_inspect/nhttp_uri_norm.h
@@ -20,6 +20,8 @@
 #ifndef NHTTP_URI_NORM_H
 #define NHTTP_URI_NORM_H
 
+#include <vector>
+
 #include "nhttp_field.h"
 #include "nhttp_module.h"
 #include "nhttp_infractions.h"
@@ -40,6 +42,7 @@ public:
         const NHttpParaList::UriParam& uri_param);
     static void classic_normalize(const Field& input, Field& result, uint8_t* buffer,
         const NHttpParaList::UriParam& uri_param);
+    static void load_default_unicode_map(uint8_t map[65536]);
 
 private:
     static bool need_norm_path(const Field& uri_component,
@@ -49,6 +52,12 @@ private:
     static int32_t norm_char_clean(const Field& input, uint8_t* out_buf,
         const NHttpParaList::UriParam& uri_param, NHttpInfractions& infractions,
         NHttpEventGen& events);
+    static int32_t norm_percent_processing(const Field& input, uint8_t* out_buf,
+        const NHttpParaList::UriParam& uri_param, bool& utf8_needed,
+        std::vector<bool>& percent_encoded, NHttpInfractions& infractions, NHttpEventGen& events);
+    static int32_t norm_utf8_processing(const Field& input, uint8_t* out_buf,
+        const NHttpParaList::UriParam& uri_param, const std::vector<bool>& percent_encoded,
+        NHttpInfractions& infractions, NHttpEventGen& events);
     static void norm_substitute(uint8_t* buf, int32_t length,
         const NHttpParaList::UriParam& uri_param,  NHttpInfractions& infractions,
         NHttpEventGen& events);
@@ -57,6 +66,8 @@ private:
     static void detect_bad_char(const Field& uri_component,
         const NHttpParaList::UriParam& uri_param, NHttpInfractions& infractions,
         NHttpEventGen& events);
+    static uint8_t reduce_to_eight_bits(uint16_t value, const NHttpParaList::UriParam& uri_param,
+        NHttpInfractions& infractions, NHttpEventGen& events);
 
     // An artifice used by the classic normalization methods to disable event generation
     class NHttpDummyEventGen : public NHttpEventGen