From: Michael Altizer (mialtize) <mialtize@cisco.com>
Date: Wed, 14 Dec 2016 23:51:48 +0000 (-0500)
Subject: Merge pull request #744 in SNORT/snort3 from nhttp58 to master
X-Git-Tag: 3.0.0-233~144
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1a42d8189349c45570812d5411349ca7139badc5;p=thirdparty%2Fsnort3.git

Merge pull request #744 in SNORT/snort3 from nhttp58 to master

Squashed commit of the following:

commit 4150d37b3afef6615b88e8350d273976290d145c
Author: allewi <allewi@cisco.com>
Date:   Mon Nov 28 20:57:00 2016 -0500

    JavaScript Normalization
---

diff --git a/src/detection/detect.cc b/src/detection/detect.cc
index eb71b1b57..1b0f1d3ef 100644
--- a/src/detection/detect.cc
+++ b/src/detection/detect.cc
@@ -76,9 +76,9 @@ void snort_ignore(Packet*) { }
 
 void snort_inspect(Packet* p)
 {
+    bool inspected = false;
     {
         PacketLatency::Context pkt_latency_ctx { p };
-        bool inspected = false;
 
         // If the packet has errors, we won't analyze it.
         if ( p->ptrs.decode_flags & DECODE_ERR_FLAGS )
@@ -121,9 +121,6 @@ void snort_inspect(Packet* p)
         if ( p->has_ip() )
             CheckTagging(p);
 
-        if ( inspected )
-            InspectorManager::clear(p);
-
         // clear closed sessions here after inspection since non-stream
         // inspectors may depend on flow information
         // FIXIT-H but this result in double clearing?  should normal
@@ -136,6 +133,9 @@ void snort_inspect(Packet* p)
     SnortEventqLog(p);
     SnortEventqReset();
 
+    if ( inspected )
+        InspectorManager::clear(p);
+
     // Handle block pending state
     Stream::check_flow_block_pending(p);
 }
diff --git a/src/pub_sub/http_events.cc b/src/pub_sub/http_events.cc
index ea0561356..d24175b75 100644
--- a/src/pub_sub/http_events.cc
+++ b/src/pub_sub/http_events.cc
@@ -71,7 +71,7 @@ const uint8_t* HttpEvent::get_referer(int32_t& length)
 
 int32_t HttpEvent::get_response_code()
 {
-    return http_msg_header->get_status_code();
+    return http_msg_header->get_status_code_num();
 }
 
 const uint8_t* HttpEvent::get_server(int32_t& length)
diff --git a/src/service_inspectors/http_inspect/CMakeLists.txt b/src/service_inspectors/http_inspect/CMakeLists.txt
index 92659dd98..756dc7792 100644
--- a/src/service_inspectors/http_inspect/CMakeLists.txt
+++ b/src/service_inspectors/http_inspect/CMakeLists.txt
@@ -60,6 +60,8 @@ set (FILE_LIST
     http_cutter.h
     http_infractions.h
     http_event_gen.h
+    http_js_norm.cc
+    http_js_norm.h
 )
 
 #if (STATIC_INSPECTORS)
diff --git a/src/service_inspectors/http_inspect/Makefile.am b/src/service_inspectors/http_inspect/Makefile.am
index f6cc10d50..f0d4db812 100644
--- a/src/service_inspectors/http_inspect/Makefile.am
+++ b/src/service_inspectors/http_inspect/Makefile.am
@@ -29,6 +29,7 @@ http_test_manager.cc http_test_manager.h \
 http_field.cc http_field.h \
 http_infractions.h \
 http_event_gen.h \
+http_js_norm.cc http_js_norm.h \
 ips_http.cc ips_http.h
 
 #if STATIC_INSPECTORS
diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h
index 3d48efce5..ca21a6280 100644
--- a/src/service_inspectors/http_inspect/http_enum.h
+++ b/src/service_inspectors/http_inspect/http_enum.h
@@ -196,6 +196,9 @@ enum Infraction
     INF_UNSUPPORTED_ENCODING,
     INF_UNKNOWN_ENCODING,
     INF_STACKED_ENCODINGS,
+    INF_JS_OBFUSCATION_EXCD,
+    INF_JS_EXCESS_WS,
+    INF_MIXED_ENCODINGS,
     INF__MAX_VALUE
 };
 
diff --git a/src/service_inspectors/http_inspect/http_field.h b/src/service_inspectors/http_inspect/http_field.h
index 95ff60092..3ab1736fb 100644
--- a/src/service_inspectors/http_inspect/http_field.h
+++ b/src/service_inspectors/http_inspect/http_field.h
@@ -47,6 +47,10 @@ public:
     // Only call this method if the field owns the dynamically allocated buffer you are deleting.
     // This method is a convenience but you still must know where the buffer came from. Many fields
     // refer to static buffers or a subfield of someone else's buffer.
+
+    // FIXIT-M the following test should undoubtedly be > 0. But this cannot be changed until a
+    // survey is done to ensure that no old code creates zero-length buffers. In practice this will
+    // happen naturally when start and length become private.
     void delete_buffer() { if (length >= 0) delete[] start; }
 
 #ifdef REG_TEST
diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc
index a093b1d75..2ea6c3657 100644
--- a/src/service_inspectors/http_inspect/http_flow_data.cc
+++ b/src/service_inspectors/http_inspect/http_flow_data.cc
@@ -21,6 +21,7 @@
 #include "http_test_manager.h"
 #include "http_flow_data.h"
 #include "http_transaction.h"
+#include "http_js_norm.h"
 
 using namespace HttpEnums;
 
diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index dd9ed9552..ff858fddc 100644
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -32,6 +32,7 @@
 #include "http_event_gen.h"
 
 class HttpTransaction;
+class HttpJsNorm;
 
 class HttpFlowData : public FlowData
 {
@@ -108,7 +109,7 @@ private:
     int64_t detect_depth_remaining[2] = { HttpEnums::STAT_NOT_PRESENT,
         HttpEnums::STAT_NOT_PRESENT };
     MimeSession* mime_state[2] = { nullptr, nullptr };
-    UtfDecodeSession* utf_state = nullptr; //SRC_SERVER only
+    UtfDecodeSession* utf_state = nullptr; // SRC_SERVER only
     uint64_t expected_trans_num[2] = { 1, 1 };
 
     // number of user data octets seen so far (regular body or chunks)
diff --git a/src/service_inspectors/http_inspect/http_js_norm.cc b/src/service_inspectors/http_inspect/http_js_norm.cc
new file mode 100644
index 000000000..ba51b1eeb
--- /dev/null
+++ b/src/service_inspectors/http_inspect/http_js_norm.cc
@@ -0,0 +1,187 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_js_norm.cc author Tom Peters <thopeter@cisco.com>
+
+#include "http_js_norm.h"
+#include "utils/util_jsnorm.h"
+#include "utils/util.h"
+#include "utils/safec.h"
+#include "http_enum.h"
+
+using namespace HttpEnums;
+
+HttpJsNorm::HttpJsNorm(int max_javascript_whitespaces_, const HttpParaList::UriParam& uri_param_) :
+    max_javascript_whitespaces(max_javascript_whitespaces_), uri_param(uri_param_),
+    javascript_search_mpse(new SearchTool()), htmltype_search_mpse(new SearchTool())
+{
+    javascript_search_mpse->add(script_start, script_start_length, JS_JAVASCRIPT);
+    javascript_search_mpse->prep();
+
+    struct HiSearchToken
+    {
+        const char* name;
+        int name_len;
+        int search_id;
+    };
+
+    const HiSearchToken html_patterns[] =
+    {
+        { "JAVASCRIPT",      10, HTML_JS },
+        { "ECMASCRIPT",      10, HTML_EMA },
+        { "VBSCRIPT",         8, HTML_VB },
+        { nullptr,            0, 0 }
+    };
+
+    for (const HiSearchToken* tmp = &html_patterns[0]; tmp->name != nullptr; tmp++)
+    {
+        htmltype_search_mpse->add(tmp->name, tmp->name_len, tmp->search_id);
+    }
+    htmltype_search_mpse->prep();
+}
+
+HttpJsNorm::~HttpJsNorm()
+{
+    delete javascript_search_mpse;
+    delete htmltype_search_mpse;
+}
+
+void HttpJsNorm::normalize(const Field& input, Field& output, bool& js_norm_alloc,
+    HttpInfractions& infractions, HttpEventGen& events) const
+{
+    bool js_present = false;
+    int index = 0;
+    const char* ptr = (const char*)input.start;
+    const char* const end = ptr + input.length;
+
+    JSState js;
+    js.allowed_spaces = max_javascript_whitespaces;
+    js.allowed_levels = MAX_ALLOWED_OBFUSCATION;
+    js.alerts = 0;
+
+    uint8_t* buffer = new uint8_t[input.length];
+
+    while (ptr < end)
+    {
+        int bytes_copied = 0;
+        int mindex;
+
+        // Search for beginning of a javascript
+        if (javascript_search_mpse->find(ptr, end-ptr, search_js_found, false, &mindex) > 0)
+        {
+            const char* js_start = ptr + mindex;
+            const char* const angle_bracket = (char*)SnortStrnStr(js_start, end - js_start, ">");
+            if (angle_bracket == nullptr)
+                break;
+
+            bool type_js = false;
+            if (angle_bracket > js_start)
+            {
+                int mid;
+                const int script_found = htmltype_search_mpse->find(
+                    js_start, (angle_bracket-js_start), search_html_found, false, &mid);
+
+                js_start = angle_bracket;
+                if (script_found > 0)
+                {
+                    switch (mid)
+                    {
+                    case HTML_JS:
+                        js_present = true;
+                        type_js = true;
+                        break;
+                    default:
+                        type_js = false;
+                        break;
+                    }
+                }
+                else
+                {
+                    // if no type or language is found we assume it is a javascript
+                    js_present = true;
+                    type_js = true;
+                }
+            }
+            // Save before the <script> begins
+            if (js_start > ptr)
+            {
+                if ((js_start - ptr) > (input.length - index))
+                    break;
+                memmove_s(buffer + index, input.length - index, ptr, js_start - ptr);
+                index += js_start - ptr;
+            }
+
+            ptr = js_start;
+            if (!type_js)
+                continue;
+
+            // FIXIT-L need to fix this library so we don't have to cast away const here.
+            JSNormalizeDecode((char*)js_start, (uint16_t)(end-js_start), (char*)buffer+index,
+                (uint16_t)(input.length - index), (char**)&ptr, &bytes_copied, &js,
+                uri_param.iis_unicode ? uri_param.unicode_map : nullptr);
+            index += bytes_copied;
+        }
+        else
+            break;
+    }
+
+    if (js_present)
+    {
+        if ((ptr < end) && ((input.length - index) >= (end - ptr)))
+        {
+            memmove_s(buffer + index, input.length - index, ptr, end - ptr); index += end - ptr;
+        }
+        if (js.alerts)
+        {
+            if (js.alerts & ALERT_LEVELS_EXCEEDED)
+            {
+                infractions += INF_JS_OBFUSCATION_EXCD;
+                events.create_event(EVENT_JS_OBFUSCATION_EXCD);
+            }
+            if (js.alerts & ALERT_SPACES_EXCEEDED)
+            {
+                infractions += INF_JS_EXCESS_WS;
+                events.create_event(EVENT_JS_EXCESS_WS);
+            }
+            if (js.alerts & ALERT_MIXED_ENCODINGS)
+            {
+                infractions += INF_MIXED_ENCODINGS;
+                events.create_event(EVENT_MIXED_ENCODINGS);
+            }
+        }
+        output.set(index, buffer);
+        js_norm_alloc = true;
+    }
+    else
+    {
+        delete[] buffer;
+        output.set(input);
+    }
+}
+
+/* Returning non-zero stops search, which is okay since we only look for one at a time */
+int HttpJsNorm::search_js_found(void*, void*, int index, void* index_ptr, void*)
+{
+    *((int*) index_ptr) = index - script_start_length;
+    return 1;
+}
+int HttpJsNorm::search_html_found(void* id, void*, int, void* id_ptr, void*)
+{
+    *((int*) id_ptr)  = (int)(uintptr_t)id;
+    return 1;
+}
+
diff --git a/src/service_inspectors/http_inspect/http_js_norm.h b/src/service_inspectors/http_inspect/http_js_norm.h
new file mode 100644
index 000000000..41e523e60
--- /dev/null
+++ b/src/service_inspectors/http_inspect/http_js_norm.h
@@ -0,0 +1,61 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_js_norm.h author Tom Peters <thopeter@cisco.com>
+
+#ifndef HTTP_JS_NORM_H
+#define HTTP_JS_NORM_H
+
+#include <cstring>
+
+#include "search_engines/search_tool.h"
+
+#include "http_field.h"
+#include "http_event_gen.h"
+#include "http_infractions.h"
+#include "http_module.h"
+
+//-------------------------------------------------------------------------
+// HttpJsNorm class
+//-------------------------------------------------------------------------
+
+class HttpJsNorm
+{
+public:
+    HttpJsNorm(int max_javascript_whitespaces_, const HttpParaList::UriParam& uri_param_);
+    ~HttpJsNorm();
+    void normalize(const Field& input, Field& output, bool& js_norm_alloc,
+        HttpInfractions& infractions, HttpEventGen& events) const;
+private:
+    enum JsSearchId { JS_JAVASCRIPT };
+    enum HtmlSearchId { HTML_JS, HTML_EMA, HTML_VB };
+
+    static constexpr const char* script_start = "<SCRIPT";
+    static constexpr int script_start_length = sizeof("<SCRIPT") - 1;
+
+    const int max_javascript_whitespaces;
+    const HttpParaList::UriParam& uri_param;
+
+    SearchTool* const javascript_search_mpse;
+    SearchTool* const htmltype_search_mpse;
+
+    static int search_js_found(void*, void*, int index, void*, void*);
+    static int search_html_found(void* id, void*, int, void*, void*);
+};
+
+#endif
+
diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc
index 7dc86d503..cb3d4da80 100644
--- a/src/service_inspectors/http_inspect/http_module.cc
+++ b/src/service_inspectors/http_inspect/http_module.cc
@@ -23,6 +23,7 @@
 #include "log/messages.h"
 
 #include "http_uri_norm.h"
+#include "http_js_norm.h"
 #include "http_module.h"
 
 using namespace HttpEnums;
@@ -34,7 +35,12 @@ const Parameter HttpModule::http_params[] =
     { "response_depth", Parameter::PT_INT, "-1:", "-1",
           "maximum response message body bytes to examine (-1 no limit)" },
     { "unzip", Parameter::PT_BOOL, nullptr, "true", "decompress gzip and deflate message bodies" },
-    { "normalize_utf", Parameter::PT_BOOL, nullptr, "true", "normalize charset utf encodings" },
+    { "normalize_utf", Parameter::PT_BOOL, nullptr, "true",
+          "normalize charset utf encodings in response bodies" },
+    { "normalize_javascript", Parameter::PT_BOOL, nullptr, "false",
+          "normalize javascript in response bodies" },
+    { "max_javascript_whitespaces", Parameter::PT_INT, "1:65535", "200",
+          "maximum consecutive whitespaces allowed within the Javascript obfuscated data" },
     { "bad_characters", Parameter::PT_BIT_LIST, "255", nullptr,
           "alert when any of specified bytes are present in URI after percent decoding" },
     { "ignore_unreserved", Parameter::PT_STRING, "(optional)", nullptr,
@@ -100,6 +106,14 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*)
     {
         params->normalize_utf = val.get_bool();
     }
+    else if (val.is("normalize_javascript"))
+    {
+        params->js_norm_param.normalize_javascript = val.get_bool();
+    }
+    else if (val.is("max_javascript_whitespaces"))
+    {
+        params->js_norm_param.max_javascript_whitespaces = val.get_long();
+    }
     else if (val.is("bad_characters"))
     {
         val.get_bits(params->uri_param.bad_characters);
@@ -206,9 +220,19 @@ bool HttpModule::end(const char*, int, SnortConfig*)
                 params->uri_param.iis_unicode_map_file.c_str(),
                 params->uri_param.iis_unicode_code_page);
     }
+    if (params->js_norm_param.normalize_javascript)
+    {
+        params->js_norm_param.js_norm =
+            new HttpJsNorm(params->js_norm_param.max_javascript_whitespaces, params->uri_param);
+    }
     return true;
 }
 
+HttpParaList::JsNormParam::~JsNormParam()
+{
+    delete js_norm;
+}
+
 // Some values in these tables may be changed by configuration parameters.
 HttpParaList::UriParam::UriParam() :
   // Characters that should not be percent-encoded
diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h
index fdd804b0f..d1d09824b 100644
--- a/src/service_inspectors/http_inspect/http_module.h
+++ b/src/service_inspectors/http_inspect/http_module.h
@@ -37,6 +37,17 @@ public:
     long response_depth;
     bool unzip;
     bool normalize_utf = true;
+
+    struct JsNormParam
+    {
+    public:
+        ~JsNormParam();
+        bool normalize_javascript = false;
+        int max_javascript_whitespaces = 200;
+        class HttpJsNorm* js_norm = nullptr;
+    };
+    JsNormParam js_norm_param;
+
     struct UriParam
     {
     public:
@@ -60,6 +71,7 @@ public:
         HttpEnums::CharAction uri_char[256];
     };
     UriParam uri_param;
+
 #ifdef REG_TEST
     bool test_input;
     bool test_output;
diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index b46f1d291..872857a92 100644
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -30,6 +30,7 @@
 #include "http_api.h"
 #include "http_msg_request.h"
 #include "http_msg_body.h"
+#include "http_js_norm.h"
 
 using namespace HttpEnums;
 
@@ -50,37 +51,30 @@ HttpMsgBody::~HttpMsgBody()
 
     if (decoded_body_alloc)
         decoded_body.delete_buffer();
+
+    if (js_norm_body_alloc)
+        js_norm_body.delete_buffer();
 }
 
 void HttpMsgBody::analyze()
 {
     do_utf_decoding(msg_text, decoded_body, decoded_body_alloc);
-    if ( decoded_body_alloc )
-    {
-        detect_data.set((decoded_body.length <= session_data->detect_depth_remaining[source_id]) ?
-            decoded_body.length : session_data->detect_depth_remaining[source_id],
-            decoded_body.start);
-    }
-    else
-    {
-        detect_data.set((msg_text.length <= session_data->detect_depth_remaining[source_id]) ?
-            msg_text.length : session_data->detect_depth_remaining[source_id],
-            msg_text.start);
-    }
 
-    session_data->detect_depth_remaining[source_id] -= detect_data.length;
-
-    // Always set file data. File processing will later set a new value in some cases.
-    file_data.set(detect_data);
-
-    if (file_data.length > 0)
+    if (session_data->detect_depth_remaining[source_id] > 0)
     {
-        set_file_data(const_cast<uint8_t*>(file_data.start), (unsigned)file_data.length);
+        do_js_normalization(decoded_body, js_norm_body, js_norm_body_alloc);
+        const int32_t detect_length =
+            (js_norm_body.length <= session_data->detect_depth_remaining[source_id]) ?
+            js_norm_body.length : session_data->detect_depth_remaining[source_id];
+        detect_data.set(detect_length, js_norm_body.start);
+        session_data->detect_depth_remaining[source_id] -= detect_length;
+        // Always set file data. File processing will later set a new value in some cases.
+        set_file_data(const_cast<uint8_t*>(detect_data.start), (unsigned)detect_data.length);
     }
 
     if (session_data->file_depth_remaining[source_id] > 0)
     {
-        do_file_processing();
+        do_file_processing(decoded_body);
     }
 
     body_octets += msg_text.length;
@@ -88,11 +82,13 @@ void HttpMsgBody::analyze()
 
 void HttpMsgBody::do_utf_decoding(const Field& input, Field& output, bool& decoded_alloc)
 {
-
-    if (!params->normalize_utf || source_id == SRC_CLIENT )
+    if (!params->normalize_utf || source_id == SRC_CLIENT)
+    {
+        output.set(input);
         return;
+    }
 
-    if (session_data->utf_state && session_data->utf_state->is_utf_encoding_present() )
+    if (session_data->utf_state && session_data->utf_state->is_utf_encoding_present())
     {
         int bytes_copied;
         bool decoded;
@@ -102,25 +98,43 @@ void HttpMsgBody::do_utf_decoding(const Field& input, Field& output, bool& decod
         if (!decoded)
         {
             delete[] buffer;
+            output.set(input);
             infractions += INF_UTF_NORM_FAIL;
             events.create_event(EVENT_UTF_NORM_FAIL);
         }
-        else if ( bytes_copied )
+        else if (bytes_copied > 0)
         {
             output.set(bytes_copied, buffer);
             decoded_alloc = true;
         }
         else
+        {
             delete[] buffer;
+            output.set(input);
+        }
     }
 
+    else
+        output.set(input);
 }
 
-void HttpMsgBody::do_file_processing()
+void HttpMsgBody::do_js_normalization(const Field& input, Field& output, bool& js_norm_alloc)
+{
+    if (!params->js_norm_param.normalize_javascript || source_id == SRC_CLIENT)
+    {
+        output.set(input);
+        return;
+    }
+
+    params->js_norm_param.js_norm->normalize(input, output, js_norm_alloc, infractions, events);
+}
+
+void HttpMsgBody::do_file_processing(Field& file_data)
 {
     // Using the trick that cutter is deleted when regular or chunked body is complete
     const bool front = (body_octets == 0);
     const bool back = (session_data->cutter[source_id] == nullptr) || tcp_close;
+
     FilePosition file_position;
     if (front && back) file_position = SNORT_FILE_FULL;
     else if (front) file_position = SNORT_FILE_START;
diff --git a/src/service_inspectors/http_inspect/http_msg_body.h b/src/service_inspectors/http_inspect/http_msg_body.h
index 30f199ec0..9877530d5 100644
--- a/src/service_inspectors/http_inspect/http_msg_body.h
+++ b/src/service_inspectors/http_inspect/http_msg_body.h
@@ -49,16 +49,18 @@ protected:
 #endif
 
 private:
-    void do_file_processing();
+    void do_file_processing(Field& file_data);
     void do_utf_decoding(const Field& input, Field& output, bool& decoded_alloc);
+    void do_js_normalization(const Field& input, Field& output, bool& js_norm_alloc);
 
     Field detect_data;
-    Field file_data;
     const bool detection_section;
     Field classic_client_body;   // URI normalization applied
     bool classic_client_body_alloc = false;
     Field decoded_body;
     bool decoded_body_alloc = false;
+    Field js_norm_body;
+    bool js_norm_body_alloc = false;
 };
 
 #endif
diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 67eae6a20..3b795fc27 100644
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -412,7 +412,6 @@ void HttpMsgHeader::setup_utf_decoding()
     session_data->utf_state->set_decode_utf_state_charset(charset_code);
 }
 
-
 #ifdef REG_TEST
 void HttpMsgHeader::print_section(FILE* output)
 {
diff --git a/src/service_inspectors/http_inspect/http_msg_header.h b/src/service_inspectors/http_inspect/http_msg_header.h
index c0e993949..3d020e126 100644
--- a/src/service_inspectors/http_inspect/http_msg_header.h
+++ b/src/service_inspectors/http_inspect/http_msg_header.h
@@ -41,11 +41,6 @@ public:
 
     void publish() override;
 
-    int32_t get_status_code()
-    {
-        return status_code_num;
-    }
-
 private:
     // Dummy configurations to support MIME processing
     MailLogConfig mime_conf;
diff --git a/src/service_inspectors/http_inspect/http_msg_section.h b/src/service_inspectors/http_inspect/http_msg_section.h
index 383838a18..510c4400e 100644
--- a/src/service_inspectors/http_inspect/http_msg_section.h
+++ b/src/service_inspectors/http_inspect/http_msg_section.h
@@ -53,6 +53,8 @@ public:
 
     HttpEnums::MethodId get_method_id() const { return method_id; }
 
+    int32_t get_status_code_num() { return status_code_num; }
+
     // Publish an inspection event for other modules to consume.
     virtual void publish() { }
 
diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc
index a53756861..cc46d4d68 100644
--- a/src/service_inspectors/http_inspect/test/http_module_test.cc
+++ b/src/service_inspectors/http_inspect/test/http_module_test.cc
@@ -25,6 +25,7 @@
 #include "service_inspectors/http_inspect/http_module.h"
 #include "service_inspectors/http_inspect/http_test_manager.h"
 #include "service_inspectors/http_inspect/http_str_to_code.h"
+#include "service_inspectors/http_inspect/http_js_norm.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
@@ -48,6 +49,11 @@ int32_t substr_to_code(const uint8_t*, const int32_t, const StrCode []) { return
 long HttpTestManager::print_amount {};
 bool HttpTestManager::print_hex {};
 
+HttpJsNorm::HttpJsNorm(int, const HttpParaList::UriParam& uri_param_) :
+    max_javascript_whitespaces(0), uri_param(uri_param_), javascript_search_mpse(nullptr),
+    htmltype_search_mpse(nullptr) {}
+HttpJsNorm::~HttpJsNorm() {}
+
 TEST_GROUP(http_peg_count_test)
 {
     HttpModule mod;
diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
index 87063ae58..5026f3d3a 100644
--- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
+++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
@@ -21,6 +21,7 @@
 
 #include "log/messages.h"
 #include "service_inspectors/http_inspect/http_uri_norm.h"
+#include "service_inspectors/http_inspect/http_js_norm.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
@@ -37,6 +38,11 @@ void show_stats(SimpleStats*, const char*) { }
 void Value::get_bits(std::bitset<256ul>&) const {}
 int SnortEventqAdd(unsigned int, unsigned int, RuleType) { return 0; }
 
+HttpJsNorm::HttpJsNorm(int, const HttpParaList::UriParam& uri_param_) :
+    max_javascript_whitespaces(0), uri_param(uri_param_), javascript_search_mpse(nullptr),
+    htmltype_search_mpse(nullptr) {}
+HttpJsNorm::~HttpJsNorm() {}
+
 TEST_GROUP(http_inspect_uri_norm)
 {
     uint8_t buffer[1000];
diff --git a/src/utils/util_jsnorm.cc b/src/utils/util_jsnorm.cc
index f0b61bcf1..11d32b8a9 100644
--- a/src/utils/util_jsnorm.cc
+++ b/src/utils/util_jsnorm.cc
@@ -96,8 +96,73 @@ typedef enum
     ACT_UNESCAPE
 } ActionJSNorm;
 
-static int hex_lookup[256];
-static int valid_chars[256];
+static const int hex_lookup[256] =
+{
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    0,               1,               2,               3,               4,               5,               6,               7,
+    8,               9,               INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, 10,              11,              12,              13,              14,              15,              INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, 10,              11,              12,              13,              14,              15,              INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+    INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL, INVALID_HEX_VAL,
+};
+
+static const int valid_chars[256] =
+{
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX,
+        IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX, IS_OCT|IS_DEC|IS_HEX,
+        IS_DEC|IS_HEX, IS_DEC|IS_HEX, 0, 0, 0, 0, 0, 0,
+
+    0, IS_HEX, IS_HEX, IS_HEX, IS_HEX, IS_HEX, IS_HEX, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, IS_HEX, IS_HEX, IS_HEX, IS_HEX, IS_HEX, IS_HEX, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
 
 static THREAD_LOCAL char decoded_out[6335];
 
@@ -374,46 +439,7 @@ static const JSNorm javascript_norm[] =
 
 static void UnescapeDecode(char*, uint16_t, char**, char**, uint16_t*, JSState*, uint8_t*);
 
-void InitJSNormLookupTable()
-{
-    int iNum;
-    int iCtr;
-
-    memset(hex_lookup, INVALID_HEX_VAL, sizeof(hex_lookup));
-    memset(valid_chars, 0, sizeof(valid_chars));
-
-    iNum = 0;
-
-    for (iCtr = 48; iCtr < 56; iCtr++)
-    {
-        hex_lookup[iCtr] = iNum;
-        valid_chars[iCtr] = (IS_HEX|IS_OCT|IS_DEC);
-        iNum++;
-    }
-
-    for (iCtr = 56; iCtr < 58; iCtr++)
-    {
-        hex_lookup[iCtr] = iNum;
-        valid_chars[iCtr] = (IS_HEX|IS_DEC);
-        iNum++;
-    }
-
-    iNum = 10;
-    for (iCtr = 65; iCtr < 71; iCtr++)
-    {
-        valid_chars[iCtr] = IS_HEX;
-        hex_lookup[iCtr] = iNum;
-        iNum++;
-    }
-
-    iNum = 10;
-    for (iCtr = 97; iCtr < 103; iCtr++)
-    {
-        valid_chars[iCtr] = IS_HEX;
-        hex_lookup[iCtr] = iNum;
-        iNum++;
-    }
-}
+void InitJSNormLookupTable() {}
 
 static inline int outBounds(const char* start, const char* end, char* ptr)
 {
diff --git a/tools/snort2lua/preprocessor_states/pps_nhttp_inspect_server.cc b/tools/snort2lua/preprocessor_states/pps_nhttp_inspect_server.cc
index a12aa7f8b..6223e9a46 100644
--- a/tools/snort2lua/preprocessor_states/pps_nhttp_inspect_server.cc
+++ b/tools/snort2lua/preprocessor_states/pps_nhttp_inspect_server.cc
@@ -126,7 +126,7 @@ bool NHttpInspectServer::convert(std::istringstream& data_stream)
             table_api.add_deleted_comment("unlimited_decompress");
 
         else if (!keyword.compare("normalize_javascript"))
-            table_api.add_deleted_comment("normalize_javascript");
+            table_api.add_option("normalize_javascript", true);
 
         else if (!keyword.compare("enable_xff"))
             table_api.add_deleted_comment("enable_xff");
@@ -229,7 +229,7 @@ bool NHttpInspectServer::convert(std::istringstream& data_stream)
             parse_deleted_option("webroot", data_stream);
 
         else if (!keyword.compare("max_javascript_whitespaces"))
-            parse_deleted_option("max_javascript_whitespaces", data_stream);
+            tmpval = parse_int_option("max_javascript_whitespaces", data_stream, false);
 
         else if (!keyword.compare("server_flow_depth") || !keyword.compare("flow_depth"))
         {
@@ -376,4 +376,3 @@ static const ConvertMap preprocessor_nhttpinpect_server =
 
 const ConvertMap* nhttpinspect_server_map = &preprocessor_nhttpinpect_server;
 }
-