Merge pull request #1007 in SNORT/snort3 from nhttp87 to master

Squashed commit of the following:

commit 811c4224c6f048f55319a95efef7402a3f079b10
Author: Tom Peters <thopeter@cisco.com>
Date:   Wed Aug 30 14:26:08 2017 -0400

    http_inspect support for u2 extra data logging

diff --git a/src/loggers/dev_notes.txt b/src/loggers/dev_notes.txt
index c2a430ae8ea7912b96906cb6717d45904ad7fc78..5c0d110f00327db204e41a17e52835cfec56809e 100644 (file)
--- a/src/loggers/dev_notes.txt
+++ b/src/loggers/dev_notes.txt
@@ -2,7 +2,7 @@ Logger subclasses that provide logging and event alerting facilities.
 
 unified2 is currently the best logger for serializing various data like
 events and packets and is the only Logger supporting extra data fields.
-Currently only the SMTP and HTTP inspectors produce exta data.
+Currently only the SMTP and HTTP inspectors produce extra data.
 
 There is separate utility called u2spewfoo provided under tools/ that can
 dump the binary u2 log in text format.

diff --git a/src/protocols/packet.h b/src/protocols/packet.h
index b5f2155835e6beae2c5b71d6a17afc6f41c3d8c8..9c4138bb1a9edf741958fb254d24c648db50f9ec 100644 (file)
--- a/src/protocols/packet.h
+++ b/src/protocols/packet.h
@@ -258,6 +258,8 @@ private:
 
 #define BIT(i) (0x1 << (i-1))
 
+inline void SetExtraData(Packet* p, const uint32_t xid) { p->xtradata_mask |= BIT(xid); }
+
 inline uint16_t extract_16bits(const uint8_t* const p)
 { return ntohs(*(uint16_t*)(p)); }
 

diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt
index f6ccfb8e749d028054a7aa341dde3a55510307a0..bdb8e53b66acbd7ec970e1a81581529b3efe8b51 100644 (file)
--- a/src/service_inspectors/http_inspect/dev_notes.txt
+++ b/src/service_inspectors/http_inspect/dev_notes.txt
@@ -2,12 +2,6 @@ The new Snort HTTP inspector (HI) is divided into two major parts. The HttpStrea
 (splitter) accepts TCP payload data from Stream and subdivides it into message sections.
 HttpInspect (inspector) processes individual message sections.
 
-Unlike other inspectors HI has an empty eval() member. All processing is done by the inspector
-process() member which is called directly from splitter reassemble(). Thus the data flow for
-processing a message section is one or more calls to splitter scan(), followed by one or more calls
-to splitter reassemble(), the last of which calls process(). The reassembled buffer returned to the
-framework is already ready for detection and the subsequent eval() call does nothing.
-
 Splitter finish() is called by the framework when the TCP connection closes (including pruning).
 It serves several specialized purposes in cases where the HTTP message is truncated (ends
 unexpectedly).

diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h
index 8e3c12c1ace89ffb81b4852a411937366927a346..e1967a09921f052e1c15be8e0af5d03316404bde 100644 (file)
--- a/src/service_inspectors/http_inspect/http_enum.h
+++ b/src/service_inspectors/http_inspect/http_enum.h
@@ -354,6 +354,7 @@ extern const bool is_sp_tab_cr_lf[256];
 extern const bool is_sp_tab_cr_lf_vt_ff[256];
 extern const bool is_sp_tab_quote_dquote[256];
 extern const bool is_print_char[256]; // printable includes SP, tab, CR, LF
+extern const bool is_sp_comma[256];
 } // end namespace HttpEnums
 
 #endif

diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc
index d2b9859fbfb6c7bcdbc8c4a0f2ca21b11a243762..2df18f2c55736b0ce7b1e77915f456727e5ba848 100644 (file)
--- a/src/service_inspectors/http_inspect/http_inspect.cc
+++ b/src/service_inspectors/http_inspect/http_inspect.cc
@@ -34,10 +34,17 @@
 #include "http_msg_status.h"
 #include "http_msg_trailer.h"
 #include "http_test_manager.h"
+#include "log/unified2.h"
 #include "protocols/packet.h"
+#include "stream/stream.h"
 
 using namespace HttpEnums;
 
+uint32_t HttpInspect::xtra_trueip_id;
+uint32_t HttpInspect::xtra_uri_id;
+uint32_t HttpInspect::xtra_host_id;
+uint32_t HttpInspect::xtra_jsnorm_id;
+
 HttpInspect::HttpInspect(const HttpParaList* params_) : params(params_)
 {
 #ifdef REG_TEST
@@ -60,6 +67,12 @@ bool HttpInspect::configure(SnortConfig* )
 {
     if (params->js_norm_param.normalize_javascript)
         params->js_norm_param.js_norm->configure();
+
+    xtra_trueip_id = Stream::reg_xtra_data_cb(get_xtra_trueip);
+    xtra_uri_id = Stream::reg_xtra_data_cb(get_xtra_uri);
+    xtra_host_id = Stream::reg_xtra_data_cb(get_xtra_host);
+    xtra_jsnorm_id = Stream::reg_xtra_data_cb(get_xtra_jsnorm);
+
     return true;
 }
 
@@ -150,6 +163,103 @@ bool HttpInspect::get_fp_buf(InspectionBuffer::Type ibt, Packet* p, InspectionBu
     return get_buf(ibt, p, b);
 }
 
+int HttpInspect::get_xtra_trueip(Flow* flow, uint8_t** buf, uint32_t* len, uint32_t* type)
+{
+    const HttpFlowData* const session_data =
+        (HttpFlowData*)flow->get_flow_data(HttpFlowData::inspector_id);
+
+    if ((session_data == nullptr) || (session_data->latest_section == nullptr))
+        return 0;
+
+    const HttpTransaction* const transaction = session_data->latest_section->get_transaction();
+    HttpMsgHeader* const req_header = transaction->get_header(SRC_CLIENT);
+    if (req_header == nullptr)
+        return 0;
+    const Field& true_ip = req_header->get_true_ip();
+    if (true_ip.length() <= 0)
+        return 0;
+
+    *buf = const_cast<uint8_t*>(true_ip.start());
+    *len = true_ip.length();
+    *type = (*len == 4) ? EVENT_INFO_XFF_IPV4 : EVENT_INFO_XFF_IPV6;
+    return 1;
+}
+
+int HttpInspect::get_xtra_uri(Flow* flow, uint8_t** buf, uint32_t* len, uint32_t* type)
+{
+    const HttpFlowData* const session_data =
+        (HttpFlowData*)flow->get_flow_data(HttpFlowData::inspector_id);
+
+    if ((session_data == nullptr) || (session_data->latest_section == nullptr))
+        return 0;
+
+    const HttpTransaction* const transaction = session_data->latest_section->get_transaction();
+    HttpMsgRequest* const request = transaction->get_request();
+    if (request == nullptr)
+        return 0;
+    const Field& uri = request->get_uri();
+    if (uri.length() <= 0)
+        return 0;
+
+    *buf = const_cast<uint8_t*>(uri.start());
+    *len = uri.length();
+    *type = EVENT_INFO_HTTP_URI;
+
+    return 1;
+}
+
+int HttpInspect::get_xtra_host(Flow* flow, uint8_t** buf, uint32_t* len, uint32_t* type)
+{
+    const HttpFlowData* const session_data =
+        (HttpFlowData*)flow->get_flow_data(HttpFlowData::inspector_id);
+
+    if ((session_data == nullptr) || (session_data->latest_section == nullptr))
+        return 0;
+
+    const HttpTransaction* const transaction = session_data->latest_section->get_transaction();
+    HttpMsgHeader* const req_header = transaction->get_header(SRC_CLIENT);
+    if (req_header == nullptr)
+        return 0;
+    const Field& host = req_header->get_header_value_norm(HEAD_HOST);
+    if (host.length() <= 0)
+        return 0;
+
+    *buf = const_cast<uint8_t*>(host.start());
+    *len = host.length();
+    *type = EVENT_INFO_HTTP_HOSTNAME;
+
+    return 1;
+}
+
+// The name of this method reflects its legacy purpose. We actually return the normalized data
+// from a response message body which may include other forms of normalization in addition to
+// JavaScript normalization. But if you don't turn JavaScript normalization on you get nothing.
+int HttpInspect::get_xtra_jsnorm(Flow* flow, uint8_t** buf, uint32_t* len, uint32_t* type)
+{
+    const HttpFlowData* const session_data =
+        (HttpFlowData*)flow->get_flow_data(HttpFlowData::inspector_id);
+
+    if ((session_data == nullptr) || (session_data->latest_section == nullptr) ||
+        (session_data->latest_section->get_source_id() != SRC_SERVER) ||
+        !session_data->latest_section->get_params()->js_norm_param.normalize_javascript)
+        return 0;
+
+    const HttpTransaction* const transaction = session_data->latest_section->get_transaction();
+    HttpMsgBody* const body = transaction->get_body();
+    if (body == nullptr)
+        return 0;
+    assert((void*)body == (void*)session_data->latest_section);
+    const Field& detect_data = body->get_detect_data();
+    if (detect_data.length() <= 0)
+        return 0;
+
+    *buf = const_cast<uint8_t*>(detect_data.start());
+    *len = detect_data.length();
+    *type = EVENT_INFO_JSNORM_DATA;
+
+    return 1;
+}
+
 void HttpInspect::eval(Packet* p)
 {
     const SourceId source_id = p->is_from_client() ? SRC_CLIENT : SRC_SERVER;
@@ -177,6 +287,13 @@ void HttpInspect::eval(Packet* p)
         }
     }
 #endif
+
+    // Whenever we process a packet we set these flags. If someone asks for an extra data
+    // buffer the JIT code will figure out if we actually have it.
+    SetExtraData(p, xtra_trueip_id);
+    SetExtraData(p, xtra_uri_id);
+    SetExtraData(p, xtra_host_id);
+    SetExtraData(p, xtra_jsnorm_id);
 }
 
 bool HttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* const flow,

diff --git a/src/service_inspectors/http_inspect/http_inspect.h b/src/service_inspectors/http_inspect/http_inspect.h
index 9484c6fbdd0c5796ccabe89f9e4ff01b9cce5fa2..96f6dfd9bb61a46b859737efb49c49cdb805b0fd 100644 (file)
--- a/src/service_inspectors/http_inspect/http_inspect.h
+++ b/src/service_inspectors/http_inspect/http_inspect.h
@@ -24,13 +24,12 @@
 // HttpInspect class
 //-------------------------------------------------------------------------
 
-#include "log/messages.h"
-
 #include "http_enum.h"
 #include "http_field.h"
 #include "http_module.h"
 #include "http_msg_section.h"
 #include "http_stream_splitter.h"
+#include "log/messages.h"
 
 class HttpApi;
 
@@ -57,6 +56,12 @@ public:
     }
     static HttpEnums::InspectSection get_latest_is(const Packet* p);
 
+    // Callbacks that provide "extra data"
+    static int get_xtra_trueip(Flow*, uint8_t**, uint32_t*, uint32_t*);
+    static int get_xtra_uri(Flow*, uint8_t**, uint32_t*, uint32_t*);
+    static int get_xtra_host(Flow*, uint8_t** buf, uint32_t* len, uint32_t* type);
+    static int get_xtra_jsnorm(Flow*, uint8_t**, uint32_t*, uint32_t*);
+
 private:
     friend HttpApi;
     friend HttpStreamSplitter;
@@ -66,6 +71,12 @@ private:
     static HttpEnums::SourceId get_latest_src(const Packet* p);
 
     const HttpParaList* const params;
+
+    // Registrations for "extra data"
+    static uint32_t xtra_trueip_id;
+    static uint32_t xtra_uri_id;
+    static uint32_t xtra_host_id;
+    static uint32_t xtra_jsnorm_id;
 };
 
 #endif

diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index 44359bfd358ab79bbb1934a4988a865435e9f4d9..facdb7762519591edc99e39baaf9cccf54316b38 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -241,10 +241,11 @@ void HttpMsgBody::do_file_processing(Field& file_data)
             {
                 if (request != nullptr)
                 {
-                    const Field& tranaction_uri = request->get_uri_norm_classic();
-                    if (tranaction_uri.length() > 0)
+                    const Field& transaction_uri = request->get_uri_norm_classic();
+                    if (transaction_uri.length() > 0)
                     {
-                        file_flows->set_file_name(tranaction_uri.start(), tranaction_uri.length());
+                        file_flows->set_file_name(transaction_uri.start(),
+                            transaction_uri.length());
                     }
                 }
             }

diff --git a/src/service_inspectors/http_inspect/http_msg_body.h b/src/service_inspectors/http_inspect/http_msg_body.h
index 87fbb66f0d811e660791c304ab101fe7382b63da..13ff23a4620655d349667656828868814f1ae963 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.h
+++ b/src/service_inspectors/http_inspect/http_msg_body.h
@@ -36,6 +36,7 @@ public:
         { return detection_section ? HttpEnums::IS_DETECTION : HttpEnums::IS_BODY; }
     bool detection_required() const override;
     const Field& get_classic_client_body();
+    const Field& get_detect_data() { return detect_data; }
     static void fd_event_callback(void* context, int event);
 
 protected:

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 5657b8020c37dba828f590684f52145256e4a417..5cb474efac075295ef81549d242301125f6c4d34 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -23,14 +23,14 @@
 
 #include "http_msg_header.h"
 
+#include "decompress/file_decomp.h"
 #include "file_api/file_flows.h"
 #include "file_api/file_service.h"
-#include "pub_sub/http_events.h"
-#include "decompress/file_decomp.h"
-
 #include "http_api.h"
 #include "http_msg_request.h"
 #include "http_msg_body.h"
+#include "pub_sub/http_events.h"
+#include "sfip/sf_ip.h"
 
 using namespace HttpEnums;
 
@@ -55,6 +55,56 @@ void HttpMsgHeader::publish()
     }
 }
 
+const Field& HttpMsgHeader::get_true_ip()
+{
+    if (true_ip.length() != STAT_NOT_COMPUTE)
+        return true_ip;
+
+    const Field* header_to_use;
+    const Field& xff = get_header_value_norm(HEAD_X_FORWARDED_FOR);
+    if (xff.length() > 0)
+        header_to_use = &xff;
+    else
+    {
+        const Field& tcip = get_header_value_norm(HEAD_TRUE_CLIENT_IP);
+        if (tcip.length() > 0)
+            header_to_use = &tcip;
+        else
+        {
+            true_ip.set(STAT_NOT_PRESENT);
+            return true_ip;
+        }
+    }
+
+    // This is potentially a comma-separated list of IP addresses. Just take the first one in
+    // the list. Since this is a normalized header field any whitespace will be an actual space.
+    uint8_t* addr_str = new uint8_t[header_to_use->length()+1];
+    int32_t length;
+    for (length = 0; length < header_to_use->length(); length++)
+    {
+        if (is_sp_comma[header_to_use->start()[length]])
+            break;
+        addr_str[length] = header_to_use->start()[length];
+    }
+    addr_str[length] = '\0';
+
+    SfIp tmp_sfip;
+    const SfIpRet status = tmp_sfip.set((char*)addr_str);
+    delete[] addr_str;
+    if (status != SFIP_SUCCESS)
+    {
+        true_ip.set(STAT_PROBLEMATIC);
+    }
+    else
+    {
+        const size_t addr_length = (tmp_sfip.is_ip6() ? 4 : 1);
+        uint32_t* const addr_buf = new uint32_t[addr_length];
+        memcpy(addr_buf, tmp_sfip.get_ptr(), addr_length * sizeof(uint32_t));
+        true_ip.set(addr_length * sizeof(uint32_t), (uint8_t*)addr_buf, true);
+    }
+    return true_ip;
+}
+
 void HttpMsgHeader::gen_events()
 {
     if ((get_header_count(HEAD_CONTENT_LENGTH) > 0) &&

diff --git a/src/service_inspectors/http_inspect/http_msg_header.h b/src/service_inspectors/http_inspect/http_msg_header.h
index 658631788e5504a97906202bd14cdc7050692f95..0733485d5716baa28db0c7372151c711c0789cc7 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_header.h
+++ b/src/service_inspectors/http_inspect/http_msg_header.h
@@ -23,6 +23,7 @@
 #include "file_api/file_api.h"
 
 #include "http_enum.h"
+#include "http_field.h"
 #include "http_msg_head_shared.h"
 
 //-------------------------------------------------------------------------
@@ -40,6 +41,7 @@ public:
     void update_flow() override;
     void gen_events() override;
     void publish() override;
+    const Field& get_true_ip();
 
 private:
     // Dummy configurations to support MIME processing
@@ -54,6 +56,8 @@ private:
 
     bool detection_section = true;
 
+    Field true_ip;
+
 #ifdef REG_TEST
     void print_section(FILE* output) override;
 #endif

diff --git a/src/service_inspectors/http_inspect/http_msg_section.h b/src/service_inspectors/http_inspect/http_msg_section.h
index f88391ab509d3c3347bccd16ca442d0dc945114a..194a225fa0788242c006dcec30da9d75d9e57e7a 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_section.h
+++ b/src/service_inspectors/http_inspect/http_msg_section.h
@@ -39,6 +39,8 @@ public:
         { return HttpEnums::IS_NONE; }
     virtual bool detection_required() const;
     HttpEnums::SourceId get_source_id() const { return source_id; }
+    HttpTransaction* get_transaction() const { return transaction; }
+    const HttpParaList* get_params() const { return params; }
 
     // Minimum necessary processing for every message
     virtual void analyze() = 0;

diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc
index b203f91de4296e23cc9ced27246c7a38f254c1c9..1c91f068ae88405b70c739bc085ca8301e330cea 100644 (file)
--- a/src/service_inspectors/http_inspect/http_tables.cc
+++ b/src/service_inspectors/http_inspect/http_tables.cc
@@ -624,6 +624,33 @@ const bool HttpEnums::is_sp_tab_quote_dquote[256] =
     false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false
 };
 
+const bool HttpEnums::is_sp_comma[256] =
+{
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+     true, false, false, false, false, false, false, false, false, false, false, false,  true, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+    false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false
+};
+
 const bool HttpEnums::is_print_char[256] =
 {
     false, false, false, false, false, false, false, false, false,  true,  true, false, false,  true, false, false,