From: Russ Combs (rucombs) <rucombs@cisco.com>
Date: Fri, 17 Apr 2020 02:51:15 +0000 (+0000)
Subject: Merge pull request #2164 in SNORT/snort3 from ~RUCOMBS/snort3:avc_only to master
X-Git-Tag: 3.0.1-2~14
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=03d8c5e8b7fcd23d1dc161d26408ae7cf781aae7;p=thirdparty%2Fsnort3.git

Merge pull request #2164 in SNORT/snort3 from ~RUCOMBS/snort3:avc_only to master

Squashed commit of the following:

commit de9579872286a2c44b89e8f1ebd4dc3b0c0593d2
Author: russ <rucombs@cisco.com>
Date:   Sat Apr 11 21:06:42 2020 -0400

    ftp_data: fix ids flushing at EOF

commit 22804f385fd0fc2eaf200fd69d7560b404700e07
Author: russ <rucombs@cisco.com>
Date:   Sat Apr 11 02:14:33 2020 -0400

    style: fix nits

commit 3248176e5aaf8132121290d791b1788db4c58469
Author: russ <rucombs@cisco.com>
Date:   Sun Apr 12 19:33:27 2020 -0400

    inspectors: designate service inspectors for start tls

    This applies to pop, imap, and smtp wich can do start tls as well as to
    ftp which can do auth tls.

commit 66a13456a5817ac628c3457365ba065f04f6b03e
Author: russ <rucombs@cisco.com>
Date:   Sat Apr 11 00:35:01 2020 -0400

    inspectors: designate service inspectors for file carving

    This applies to dce_smb, ftp_data, http_inspect, http2_inspect, imap,
    pop, and smtp which process files.

commit 3700b32c0d3e596bdea00beb321edb4a992533b4
Author: russ <rucombs@cisco.com>
Date:   Sat Apr 11 00:33:39 2020 -0400

    inspectors: designate service inspectors control channels for avc only

    This applies to cip, ftp_server, and sip inspectors which support other
    flows.

commit 0749035648aaee49de05e77100b268a23b89b484
Author: russ <rucombs@cisco.com>
Date:   Sat Apr 11 00:18:26 2020 -0400

    ips_context: add support to fallback to avc only
---

diff --git a/src/codecs/ip/cd_gre.cc b/src/codecs/ip/cd_gre.cc
index d25e3a02c..41f0e4da9 100644
--- a/src/codecs/ip/cd_gre.cc
+++ b/src/codecs/ip/cd_gre.cc
@@ -136,7 +136,7 @@ bool GreCodec::encode(const uint8_t* const raw_in, const uint16_t raw_len,
     if (GRE_SEQ(greh_out))
     {
         uint16_t len = 4; // Flags, version and protocol
-        
+
         if (GRE_CHKSUM(greh_out))
             len += 4;
 
@@ -151,7 +151,7 @@ bool GreCodec::encode(const uint8_t* const raw_in, const uint16_t raw_len,
         assert(raw_len >= 6);
         // Checksum field is zero for computing checksum
         *(uint16_t*)(buf.data() + 4) = 0;
-        *(uint16_t*)(buf.data() + 4) = checksum::cksum_add((uint16_t*)buf.data(), 
+        *(uint16_t*)(buf.data() + 4) = checksum::cksum_add((uint16_t*)buf.data(),
             buf.size());
     }
 
diff --git a/src/detection/context_switcher.cc b/src/detection/context_switcher.cc
index ed3625199..686be66bf 100644
--- a/src/detection/context_switcher.cc
+++ b/src/detection/context_switcher.cc
@@ -83,6 +83,8 @@ void ContextSwitcher::start()
     c->packet->action = &c->packet->action_inst;
     c->state = IpsContext::BUSY;
 
+    c->setup();
+
     busy.emplace_back(c);
 }
 
@@ -98,7 +100,8 @@ void ContextSwitcher::stop()
 		"(wire) %" PRIu64 " cs::stop %" PRIu64 " (i=%zu, b=%zu)\n",
         get_packet_number(), c->context_num, idle.size(), busy.size());
 
-    c->clear_context_data();
+    c->clear();
+
     c->packet->active = nullptr;
     c->packet->action = nullptr;
     c->state = IpsContext::IDLE;
@@ -137,7 +140,7 @@ void ContextSwitcher::abort()
         c->abort();
         c->state = IpsContext::IDLE;
         c->clear_callbacks();
-        c->clear_context_data();
+        c->clear();
         idle.emplace_back(c);
     }
     non_flow_chain.abort();
@@ -160,6 +163,8 @@ IpsContext* ContextSwitcher::interrupt()
     idle.pop_back();
 
     c->state = IpsContext::BUSY;
+    c->setup();
+
     busy.emplace_back(c);
     return c;
 }
@@ -178,7 +183,7 @@ IpsContext* ContextSwitcher::complete()
         c->packet_number, c->context_num, idle.size(), busy.size());
 
     busy.pop_back();
-    c->clear_context_data();
+    c->clear();
     c->state = IpsContext::IDLE;
     idle.emplace_back(c);
 
diff --git a/src/detection/ips_context.cc b/src/detection/ips_context.cc
index 69cebb4de..a5752c616 100644
--- a/src/detection/ips_context.cc
+++ b/src/detection/ips_context.cc
@@ -32,6 +32,7 @@
 #include "events/event_queue.h"
 #include "events/sfeventq.h"
 #include "main/snort_config.h"
+#include "stream/stream.h"
 
 #ifdef UNIT_TEST
 #include "catch/snort_catch.h"
@@ -88,6 +89,31 @@ IpsContext::~IpsContext()
     delete packet;
 }
 
+void IpsContext::setup()
+{
+    conf = SnortConfig::get_conf();
+    remove_gadget = false;
+}
+
+void IpsContext::clear()
+{
+    for ( auto id : ids_in_use )
+    {
+        auto* p = data[id];
+        if ( p )
+            p->clear();
+    }
+    if ( remove_gadget and packet->flow and !packet->is_rebuilt() )
+    {
+       Stream::disable_reassembly(packet->flow);
+
+       if ( packet->flow->gadget )
+           packet->flow->clear_gadget();
+    }
+    remove_gadget = false;
+    assert(post_callbacks.empty());
+}
+
 void IpsContext::set_context_data(unsigned id, IpsContextData* cd)
 {
     assert(id < data.size());
@@ -101,16 +127,6 @@ IpsContextData* IpsContext::get_context_data(unsigned id) const
     return data[id];
 }
 
-void IpsContext::clear_context_data()
-{
-    for ( auto id : ids_in_use )
-    {
-        auto* p = data[id];
-        if ( p )
-            p->clear();
-    }
-}
-
 void IpsContext::snapshot_flow(Flow* f)
 {
     flow.session_flags = f->ssn_state.session_flags;
@@ -125,6 +141,26 @@ void IpsContext::post_detection()
     post_callbacks.clear();
 }
 
+void IpsContext::disable_detection()
+{
+    if ( active_rules == IpsContext::NONE )
+        return;
+
+    IpsPolicy* empty_policy = get_empty_ips_policy(conf);
+    set_ips_policy(empty_policy);
+
+    if ( Flow* f = packet->flow )
+    {
+        f->ips_policy_id = empty_policy->policy_id;
+        f->set_to_client_detection(false);
+        f->set_to_server_detection(false);
+    }
+    active_rules = IpsContext::NONE;
+}
+
+void IpsContext::disable_inspection()
+{ remove_gadget = true; }
+
 //--------------------------------------------------------------------------
 // unit tests
 //--------------------------------------------------------------------------
diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h
index 409b41773..885f5de1c 100644
--- a/src/detection/ips_context.h
+++ b/src/detection/ips_context.h
@@ -67,9 +67,11 @@ public:
     IpsContext(const IpsContext&) = delete;
     IpsContext& operator=(const IpsContext&) = delete;
 
+    void setup();
+    void clear();
+
     void set_context_data(unsigned id, IpsContextData*);
     IpsContextData* get_context_data(unsigned id) const;
-    void clear_context_data();
 
     void snapshot_flow(Flow*);
 
@@ -79,6 +81,9 @@ public:
     SnortProtocolId get_snort_protocol_id()
     { return flow.proto_id; }
 
+    void disable_detection();
+    void disable_inspection();
+
     enum ActiveRules
     { NONE, NON_CONTENT, CONTENT };
 
@@ -170,6 +175,7 @@ private:
     std::vector<Callback> post_callbacks;
     IpsContext* depends_on;
     IpsContext* next_to_process;
+    bool remove_gadget;
 };
 }
 #endif
diff --git a/src/flow/session.h b/src/flow/session.h
index 1f5a37a1b..0fcd598fb 100644
--- a/src/flow/session.h
+++ b/src/flow/session.h
@@ -72,8 +72,10 @@ public:
     virtual bool is_sequenced(uint8_t /*dir*/) { return true; }
     virtual bool are_packets_missing(uint8_t /*dir*/) { return true; }
 
+    virtual void disable_reassembly(snort::Flow*) { }
     virtual uint8_t get_reassembly_direction() { return SSN_DIR_NONE; }
     virtual uint8_t missing_in_reassembled(uint8_t /*dir*/) { return SSN_MISSING_NONE; }
+
     virtual bool set_packet_action_to_hold(snort::Packet*) { return false; }
 
 protected:
diff --git a/src/framework/inspector.h b/src/framework/inspector.h
index d6dffc84c..a53c2a134 100644
--- a/src/framework/inspector.h
+++ b/src/framework/inspector.h
@@ -102,9 +102,7 @@ public:
     bool is_inactive();
 
     void set_service(SnortProtocolId snort_protocol_id_param)
-    {
-        snort_protocol_id = snort_protocol_id_param;
-    }
+    { snort_protocol_id = snort_protocol_id_param; }
 
     SnortProtocolId get_service() { return snort_protocol_id; }
 
@@ -136,6 +134,15 @@ public:
 
     const char* get_name();
 
+    virtual bool is_control_channel() const
+    { return false; }
+
+    virtual bool can_carve_files() const
+    { return false; }
+
+    virtual bool can_start_tls() const
+    { return false; }
+
 public:
     static unsigned max_slots;
     static THREAD_LOCAL unsigned slot;
diff --git a/src/network_inspectors/perf_monitor/perf_module.h b/src/network_inspectors/perf_monitor/perf_module.h
index c6a2aa8be..5782b0873 100644
--- a/src/network_inspectors/perf_monitor/perf_module.h
+++ b/src/network_inspectors/perf_monitor/perf_module.h
@@ -99,7 +99,7 @@ struct PerfConfig
     std::vector<snort::Module*> mods_to_prep;
     PerfConstraints* constraints;
 
-    PerfConfig() { constraints = new PerfConstraints; } 
+    PerfConfig() { constraints = new PerfConstraints; }
     ~PerfConfig() { delete constraints; }
 
     bool resolve();
diff --git a/src/service_inspectors/cip/cip.cc b/src/service_inspectors/cip/cip.cc
index 4ccc0504b..4ffc824cc 100644
--- a/src/service_inspectors/cip/cip.cc
+++ b/src/service_inspectors/cip/cip.cc
@@ -256,9 +256,10 @@ public:
     void eval(Packet*) override;
 
     class StreamSplitter* get_splitter(bool c2s) override
-    {
-        return new CipSplitter(c2s);
-    }
+    { return new CipSplitter(c2s); }
+
+    bool is_control_channel() const override
+    { return true; }
 
 private:
     CipProtoConf* config;
diff --git a/src/service_inspectors/dce_rpc/dce_smb.cc b/src/service_inspectors/dce_rpc/dce_smb.cc
index 4b165a573..9fc6e1465 100644
--- a/src/service_inspectors/dce_rpc/dce_smb.cc
+++ b/src/service_inspectors/dce_rpc/dce_smb.cc
@@ -324,10 +324,12 @@ public:
     void show(SnortConfig*) override;
     void eval(Packet*) override;
     void clear(Packet*) override;
+
     StreamSplitter* get_splitter(bool c2s) override
-    {
-        return new Dce2SmbSplitter(c2s);
-    }
+    { return new Dce2SmbSplitter(c2s); }
+
+    bool can_carve_files() const override
+    { return true; }
 
 private:
     dce2SmbProtoConf config;
diff --git a/src/service_inspectors/ftp_telnet/ftp.cc b/src/service_inspectors/ftp_telnet/ftp.cc
index 6ece3faa0..b89a4bb80 100644
--- a/src/service_inspectors/ftp_telnet/ftp.cc
+++ b/src/service_inspectors/ftp_telnet/ftp.cc
@@ -217,6 +217,12 @@ public:
     void eval(Packet*) override;
     StreamSplitter* get_splitter(bool) override;
 
+    bool is_control_channel() const override
+    { return true; }
+
+    bool can_start_tls() const override
+    { return true; }
+
     FTP_SERVER_PROTO_CONF* ftp_server;
 };
 
diff --git a/src/service_inspectors/ftp_telnet/ftp_data.cc b/src/service_inspectors/ftp_telnet/ftp_data.cc
index a5f05d044..a75d8a0f5 100644
--- a/src/service_inspectors/ftp_telnet/ftp_data.cc
+++ b/src/service_inspectors/ftp_telnet/ftp_data.cc
@@ -245,6 +245,9 @@ public:
 
     void eval(Packet*) override;
     StreamSplitter* get_splitter(bool to_server) override;
+
+    bool can_carve_files() const override
+    { return true; }
 };
 
 class FtpDataModule : public Module
diff --git a/src/service_inspectors/ftp_telnet/ftpdata_splitter.h b/src/service_inspectors/ftp_telnet/ftpdata_splitter.h
index cb3b1d9a7..83aa87f1a 100644
--- a/src/service_inspectors/ftp_telnet/ftpdata_splitter.h
+++ b/src/service_inspectors/ftp_telnet/ftpdata_splitter.h
@@ -36,10 +36,12 @@ public:
         expected_seg_size = 0;
     }
 
-
     Status scan(snort::Packet*, const uint8_t*, uint32_t len, uint32_t flags, uint32_t* fp ) override;
     bool finish(snort::Flow*) override;
 
+    bool is_paf() override
+    { return true; }
+
 private:
     uint16_t min;
     uint16_t segs;
diff --git a/src/service_inspectors/http2_inspect/http2_inspect.h b/src/service_inspectors/http2_inspect/http2_inspect.h
index 38db7caa8..10904e975 100644
--- a/src/service_inspectors/http2_inspect/http2_inspect.h
+++ b/src/service_inspectors/http2_inspect/http2_inspect.h
@@ -47,10 +47,12 @@ public:
     bool configure(snort::SnortConfig*) override;
     void eval(snort::Packet* p) override;
     void clear(snort::Packet* p) override;
+
     Http2StreamSplitter* get_splitter(bool is_client_to_server) override
-    {
-        return new Http2StreamSplitter(is_client_to_server);
-    }
+    { return new Http2StreamSplitter(is_client_to_server); }
+
+    bool can_carve_files() const override
+    { return true; }
 
 private:
     friend Http2Api;
diff --git a/src/service_inspectors/http_inspect/http_inspect.h b/src/service_inspectors/http_inspect/http_inspect.h
index 356ecda7a..1c51e1474 100644
--- a/src/service_inspectors/http_inspect/http_inspect.h
+++ b/src/service_inspectors/http_inspect/http_inspect.h
@@ -54,13 +54,16 @@ public:
     void show(snort::SnortConfig*) override;
     void eval(snort::Packet* p) override;
     void clear(snort::Packet* p) override;
+
     HttpStreamSplitter* get_splitter(bool is_client_to_server) override
-    {
-        return new HttpStreamSplitter(is_client_to_server, this);
-    }
+    { return new HttpStreamSplitter(is_client_to_server, this); }
+
+    bool can_carve_files() const override
+    { return true; }
+
     static HttpEnums::InspectSection get_latest_is(const snort::Packet* p);
     static HttpCommon::SourceId get_latest_src(const snort::Packet* p);
-    void disable_detection(snort::Packet *p);
+    void disable_detection(snort::Packet* p);
 
     // Callbacks that provide "extra data"
     static int get_xtra_trueip(snort::Flow*, uint8_t**, uint32_t*, uint32_t*);
diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
index 04b408142..49be2d494 100644
--- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
@@ -362,10 +362,12 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total,
 
     HttpModule::increment_peg_counts(PEG_REASSEMBLE);
 
-    const bool is_body = (session_data->section_type[source_id] == SEC_BODY_CHUNK) ||
-                         (session_data->section_type[source_id] == SEC_BODY_CL) ||
-                         (session_data->section_type[source_id] == SEC_BODY_OLD) ||
-                         (session_data->section_type[source_id] == SEC_BODY_H2);
+    const bool is_body =
+        (session_data->section_type[source_id] == SEC_BODY_CHUNK) ||
+        (session_data->section_type[source_id] == SEC_BODY_CL) ||
+        (session_data->section_type[source_id] == SEC_BODY_OLD) ||
+        (session_data->section_type[source_id] == SEC_BODY_H2);
+
     uint8_t*& buffer = session_data->section_buffer[source_id];
     if (buffer == nullptr)
     {
diff --git a/src/service_inspectors/imap/imap.cc b/src/service_inspectors/imap/imap.cc
index 87b596b97..001f8e9c0 100644
--- a/src/service_inspectors/imap/imap.cc
+++ b/src/service_inspectors/imap/imap.cc
@@ -700,6 +700,12 @@ public:
     StreamSplitter* get_splitter(bool c2s) override
     { return new ImapSplitter(c2s); }
 
+    bool can_carve_files() const override
+    { return true; }
+
+    bool can_start_tls() const override
+    { return true; }
+
 private:
     IMAP_PROTO_CONF* config;
 };
diff --git a/src/service_inspectors/pop/pop.cc b/src/service_inspectors/pop/pop.cc
index 9eac981c7..4819da9e8 100644
--- a/src/service_inspectors/pop/pop.cc
+++ b/src/service_inspectors/pop/pop.cc
@@ -641,6 +641,12 @@ public:
     StreamSplitter* get_splitter(bool c2s) override
     { return new PopSplitter(c2s); }
 
+    bool can_carve_files() const override
+    { return true; }
+
+    bool can_start_tls() const override
+    { return true; }
+
 private:
     POP_PROTO_CONF* config;
 };
diff --git a/src/service_inspectors/sip/sip.cc b/src/service_inspectors/sip/sip.cc
index 74e916845..6902a05f9 100644
--- a/src/service_inspectors/sip/sip.cc
+++ b/src/service_inspectors/sip/sip.cc
@@ -199,6 +199,9 @@ public:
     class StreamSplitter* get_splitter(bool to_server) override
     { return new SipSplitter(to_server); }
 
+    bool is_control_channel() const override
+    { return true; }
+
 private:
     SIP_PROTO_CONF* config;
 };
diff --git a/src/service_inspectors/smtp/smtp.cc b/src/service_inspectors/smtp/smtp.cc
index 3aea12cef..c6b0eb04e 100644
--- a/src/service_inspectors/smtp/smtp.cc
+++ b/src/service_inspectors/smtp/smtp.cc
@@ -1410,6 +1410,12 @@ public:
     StreamSplitter* get_splitter(bool c2s) override
     { return new SmtpSplitter(c2s, config->max_auth_command_line_len); }
 
+    bool can_carve_files() const override
+    { return true; }
+
+    bool can_start_tls() const override
+    { return true; }
+
     void ProcessSmtpCmdsList(const SmtpCmd*);
 
 private:
diff --git a/src/stream/stream.cc b/src/stream/stream.cc
index b017e9d76..c3ef74196 100644
--- a/src/stream/stream.cc
+++ b/src/stream/stream.cc
@@ -737,6 +737,12 @@ void Stream::set_extra_data(
     flow->session->set_extra_data(p, flag);
 }
 
+void Stream::disable_reassembly(Flow* flow)
+{
+    assert(flow && flow->session);
+    return flow->session->disable_reassembly(flow);
+}
+
 char Stream::get_reassembly_direction(Flow* flow)
 {
     assert(flow && flow->session);
diff --git a/src/stream/stream.h b/src/stream/stream.h
index 50cd483c0..1a893c189 100644
--- a/src/stream/stream.h
+++ b/src/stream/stream.h
@@ -143,7 +143,7 @@ public:
         Flow*, Packet* p, uint32_t gid, uint32_t sid,
         uint32_t eventId, uint32_t eventSecond);
 
-    // Get reassembly direction for given session
+    static void disable_reassembly(Flow*);
     static char get_reassembly_direction(Flow*);
 
     // Returns true if stream data for the flow is in sequence, otherwise return false.
diff --git a/src/stream/tcp/tcp_stream_session.cc b/src/stream/tcp/tcp_stream_session.cc
index ef929c9d4..a5b2cbe81 100644
--- a/src/stream/tcp/tcp_stream_session.cc
+++ b/src/stream/tcp/tcp_stream_session.cc
@@ -139,6 +139,21 @@ void TcpStreamSession::set_no_ack(bool b)
     }
 }
 
+void TcpStreamSession::disable_reassembly(Flow* f)
+{
+    client.set_splitter((StreamSplitter*)nullptr);
+    server.set_splitter((StreamSplitter*)nullptr);
+
+    client.reassembler.purge_segment_list();
+    server.reassembler.purge_segment_list();
+
+    client.flush_policy = STREAM_FLPOLICY_IGNORE;
+    server.flush_policy = STREAM_FLPOLICY_IGNORE;
+
+    client.finalize_held_packet(f);
+    server.finalize_held_packet(f);
+}
+
 uint8_t TcpStreamSession::get_reassembly_direction()
 {
     uint8_t dir = SSN_DIR_NONE;
diff --git a/src/stream/tcp/tcp_stream_session.h b/src/stream/tcp/tcp_stream_session.h
index eb4dc54b9..aaf2d4d71 100644
--- a/src/stream/tcp/tcp_stream_session.h
+++ b/src/stream/tcp/tcp_stream_session.h
@@ -29,7 +29,7 @@
 #include "tcp_stream_config.h"
 #include "tcp_stream_tracker.h"
 
-// FIXIT-L session tracking must be split from reassembly
+// FIXIT-L session tracking could be split from reassembly
 // into a separate module a la ip_session.cc and ip_defrag.cc
 // (of course defrag should also be cleaned up)
 class TcpStreamSession : public Session
@@ -40,16 +40,22 @@ public:
     bool setup(snort::Packet*) override;
     void clear() override;
     void cleanup(snort::Packet* = nullptr) override;
+
     void set_splitter(bool, snort::StreamSplitter*) override;
     snort::StreamSplitter* get_splitter(bool) override;
+
     bool is_sequenced(uint8_t /*dir*/) override;
     bool are_packets_missing(uint8_t /*dir*/) override;
+
+    void disable_reassembly(snort::Flow*) override;
     uint8_t get_reassembly_direction() override;
     uint8_t missing_in_reassembled(uint8_t /*dir*/) override;
+
     bool add_alert(snort::Packet*, uint32_t gid, uint32_t sid) override;
     bool check_alerted(snort::Packet*, uint32_t gid, uint32_t sid) override;
     int update_alert(snort::Packet*, uint32_t /*gid*/, uint32_t /*sid*/,
         uint32_t /*event_id*/, uint32_t /*event_second*/) override;
+
     bool set_packet_action_to_hold(snort::Packet*) override;
 
     uint16_t get_mss(bool to_server) const;
diff --git a/src/stream/tcp/tcp_stream_tracker.cc b/src/stream/tcp/tcp_stream_tracker.cc
index b2e078c49..a0368c55d 100644
--- a/src/stream/tcp/tcp_stream_tracker.cc
+++ b/src/stream/tcp/tcp_stream_tracker.cc
@@ -509,7 +509,7 @@ void TcpStreamTracker::update_tracker_ack_sent(TcpSegmentDescriptor& tsd)
     }
 
     snd_wnd = tsd.get_seg_wnd();
-    reassembler.flush_on_ack_policy(tsd.get_pkt() );
+    reassembler.flush_on_ack_policy(tsd.get_pkt());
 }
 
 bool TcpStreamTracker::update_on_3whs_ack(TcpSegmentDescriptor& tsd)