Pull request #4365: extractor: add flow id

Merge in SNORT/snort3 from ~ANOROKH/snort3:extractor_flow_id to master

Squashed commit of the following:

commit 295a374f5a2616be85946d029150f3e9faf04447
Author: anorokh <anorokh@cisco.com>
Date:   Fri Jun 14 12:01:28 2024 +0300

    extractor: add flow hash key

      * updated conf parsing logic
      * updated Error messages

diff --git a/src/network_inspectors/extractor/dev_notes.txt b/src/network_inspectors/extractor/dev_notes.txt
index 5f40acbbd95ad569a066e31c2df981cc50aa0967..2878b160f5fc1e5dd8a910dd50440e9fe414c661 100644 (file)
--- a/src/network_inspectors/extractor/dev_notes.txt
+++ b/src/network_inspectors/extractor/dev_notes.txt
@@ -31,6 +31,7 @@ Each tenant can have its own protocol configuration.
 
 A list of common fields which are logged:
  * ts (timestamp)
+ * uid (connection id)
  * id.orig_h (client IP address)
  * id.orig_p (client TCP port)
  * id.resp_h (server IP address)

diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc
index 1234b525daf04dc1663a9451660f2ca58fd1f92a..b569f5005efd659eedbabc5b6c45dbf76b671892 100644 (file)
--- a/src/network_inspectors/extractor/extractor.cc
+++ b/src/network_inspectors/extractor/extractor.cc
@@ -92,10 +92,7 @@ void ExtractorModule::commit_config()
     for (const auto& p : extractor_config.protocols)
     {
         if (p.tenant_id == service_config.tenant_id and p.service == service_config.service)
-        {
-            ParseError("%s service got multiple configurations", service_config.service.c_str());
-            break;
-        }
+            ParseWarning(WARN_CONF_STRICT, "%s service got multiple configurations", service_config.service.c_str());
     }
 
     extractor_config.protocols.push_back(service_config);
@@ -156,16 +153,17 @@ bool ExtractorModule::set(const char*, Value& v, SnortConfig*)
 
 bool ExtractorModule::end(const char* fqn, int idx, SnortConfig*)
 {
-    if (idx > 0 && !strcmp(fqn, "extractor.protocols"))
+    if (!idx or strcmp(fqn, "extractor.protocols"))
+        return true;
+
+    if (service_config.fields.empty())
     {
-        if (service_config.fields.empty())
-        {
-            ParseError("Can't initialize extractor without protocols.fields");
-            return false;
-        }
-        commit_config();
+        ParseError("can't initialize extractor without protocols.fields");
+        return false;
     }
 
+    commit_config();
+
     return true;
 }
 

diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc
index 91800204ee3006517e71449b7d9493f65528f9cb..4b9c96960ce1baf2a3f0d18e2b5b46c2bbb35393 100644 (file)
--- a/src/network_inspectors/extractor/extractor_csv_logger.cc
+++ b/src/network_inspectors/extractor/extractor_csv_logger.cc
@@ -17,6 +17,10 @@
 //--------------------------------------------------------------------------
 // csv_logger.cc author Anna Norokh <anorokh@cisco.com>
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include "extractor_csv_logger.h"
 
 #include <algorithm>

diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h
index abb81526bfb495847c22cb894f85df950f26d5ed..e996ec44114b393a7e1322cb839e6a713412c374 100644 (file)
--- a/src/network_inspectors/extractor/extractor_csv_logger.h
+++ b/src/network_inspectors/extractor/extractor_csv_logger.h
@@ -30,7 +30,11 @@ class CsvExtractorLogger : public ExtractorLogger
 public:
     CsvExtractorLogger(OutputType o_type, const std::vector<std::string>& fields)
         : ExtractorLogger(fields), writer(ExtractorWriter::make_writer(o_type))
-    { CsvExtractorLogger::add_header(); }
+    {
+        if (writer)
+            CsvExtractorLogger::add_header();
+    }
+
     ~CsvExtractorLogger() override;
 
     void add_header() override;

diff --git a/src/network_inspectors/extractor/extractor_event_handlers.h b/src/network_inspectors/extractor/extractor_event_handlers.h
index ff0b490deab9ce70f334f75e841dbc72cdae21c7..038db11765c24bb3ea762784ca2b8e1b97e2059b 100644 (file)
--- a/src/network_inspectors/extractor/extractor_event_handlers.h
+++ b/src/network_inspectors/extractor/extractor_event_handlers.h
@@ -20,6 +20,7 @@
 #ifndef EXTRACTOR_EVENT_HANDLERS_H
 #define EXTRACTOR_EVENT_HANDLERS_H
 
+#include "flow/flow_key.h"
 #include "framework/data_bus.h"
 
 #include "extractor.h"
@@ -30,6 +31,13 @@ namespace snort
 
 class ExtractorEvent
 {
+public:
+    static FlowHashKeyOps& get_hash()
+    {
+        static thread_local FlowHashKeyOps flow_key_ops(0);
+        return flow_key_ops;
+    }
+
 protected:
     ExtractorEvent(uint32_t tid, const std::vector<std::string>& flds, ExtractorLogger& l)
         : tenant_id(tid), fields(flds), logger(l) {}

diff --git a/src/network_inspectors/extractor/extractor_http_event_handler.cc b/src/network_inspectors/extractor/extractor_http_event_handler.cc
index 22d8232f64bc0ad6be25d5f3e0f380985f11ce28..b76c50f46a10f4faade4939082d8561b5bd2c1b5 100644 (file)
--- a/src/network_inspectors/extractor/extractor_http_event_handler.cc
+++ b/src/network_inspectors/extractor/extractor_http_event_handler.cc
@@ -57,6 +57,7 @@ Value* get_ip_dst(DataEvent*, Packet*, Flow*);
 Value* get_ip_src_port(DataEvent*, Packet*, Flow*);
 Value* get_ip_dst_port(DataEvent*, Packet*, Flow*);
 Value* get_pkt_num(DataEvent*, Packet*, Flow*);
+Value* get_uid(DataEvent*, Packet*, Flow*);
 
 static void field_to_string(const Field& field, std::string& value)
 {
@@ -187,9 +188,17 @@ Value* get_pkt_num(DataEvent*, Packet* p, Flow*)
     return new Value(p->context->packet_number);
 }
 
+Value* get_uid(DataEvent*, Packet*, Flow* f)
+{
+    unsigned key = ExtractorEvent::get_hash().do_hash((const unsigned char*)f->key, 0);
+
+    return new Value((uint64_t)key);
+}
+
 static std::map<std::string, GetFunc> event_getters =
 {
     {"ts", get_timestamp},
+    {"uid", get_uid},
     {"id.orig_h", get_ip_src},
     {"id.resp_h", get_ip_dst},
     {"id.orig_p", get_ip_src_port},

diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc
index b49abfdd6daad77386112eb912873bd07a08f079..fa8d7cae17641c1eea9fd0fc173fb16972702c96 100644 (file)
--- a/src/network_inspectors/extractor/extractor_logger.cc
+++ b/src/network_inspectors/extractor/extractor_logger.cc
@@ -17,18 +17,32 @@
 //--------------------------------------------------------------------------
 // extractor_logger.cc author Anna Norokh <anorokh@cisco.com>
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include "extractor_logger.h"
 
+#include <cassert>
+
 #include "extractor_csv_logger.h"
 
 ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type,
     const std::vector<std::string>& fields)
 {
+    if (fields.empty())
+        return nullptr;
+
+    ExtractorLogger* logger = nullptr;
+
     switch (f_type)
     {
     case FormatType::CSV:
-        return new CsvExtractorLogger(o_type, fields);
+        logger = new CsvExtractorLogger(o_type, fields);
+        break;
     }
 
-    return nullptr;
+    assert(logger);
+
+    return logger;
 }

diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc
index 11c0ebc76cbbfe321509dbba78195f6412615b06..e9e1fabe5651778a51570b299581d39da3058c52 100644 (file)
--- a/src/network_inspectors/extractor/extractor_service.cc
+++ b/src/network_inspectors/extractor/extractor_service.cc
@@ -40,6 +40,7 @@ using namespace snort;
 std::vector<std::string> ExtractorService::common_fields =
 {
     "ts",
+    "uid",
     "id.orig_h",
     "id.orig_p",
     "id.resp_h",
@@ -54,7 +55,6 @@ ExtractorService::ExtractorService(uint32_t tenant, const std::vector<std::strin
 {
     add_fields(srv_fields);
     add_events(srv_events);
-
     logger = ExtractorLogger::make_logger(f_type, o_type, get_fields());
 }
 
@@ -65,7 +65,7 @@ void ExtractorService::add_events(const std::vector<std::string>& vals)
         if (find_event(val))
             events.push_back(val);
         else
-            ParseError("Invalid protocols.on_events value %s", val.c_str());
+            ParseWarning(WARN_CONF_STRICT, "unsupported '%s' event in protocols.on_events", val.c_str());
     }
 }
 
@@ -76,28 +76,32 @@ void ExtractorService::add_fields(const std::vector<std::string>& vals)
         if (find_field(val))
             fields.push_back(val);
         else
-            ParseError("Invalid protocols.fields value %s", val.c_str());
+            ParseWarning(WARN_CONF_STRICT, "unsupported '%s' field in protocols.fields", val.c_str());
     }
 }
 
 ExtractorService* ExtractorService::make_service(const ServiceConfig& cfg, FormatType f_type, OutputType o_type)
 {
     if (cfg.on_events.empty())
+    {
         ParseError("%s service misses on_events field", cfg.service.c_str());
+        return nullptr;
+    }
+
+    ExtractorService* srv = nullptr;
 
     switch (cfg.service)
     {
-        case ServiceType::HTTP:
-            return new HttpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, f_type, o_type);
+    case ServiceType::HTTP:
+        srv = new HttpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, f_type, o_type);
+        break;
 
-        case ServiceType::UNDEFINED:
-            ParseError("%s service is not supported", cfg.service.c_str());
-            break;
-
-        default:
-            return nullptr;
+    case ServiceType::UNDEFINED: // fallthrough
+    default:
+        ParseError("'%s' service is not supported", cfg.service.c_str());
     }
-    return nullptr;
+
+    return srv;
 }
 
 bool ExtractorService::find_event(const std::string& event) const
@@ -163,6 +167,9 @@ HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector<st
     const std::vector<std::string>& srv_events, ServiceType s_type, FormatType f_type, OutputType o_type)
     : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, f_type, o_type)
 {
+    if (!logger)
+        return;
+
     for (const auto& event : get_events())
     {
         if (!strcmp("eot", event.c_str()))
@@ -172,4 +179,3 @@ HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector<st
         }
     }
 }
-

diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h
index 0afeb12b9ac118f22a0e69029be05a37a1a52883..2bb560ea17d6bb1d3991a5203cca44fc00f2f63a 100644 (file)
--- a/src/network_inspectors/extractor/extractor_service.h
+++ b/src/network_inspectors/extractor/extractor_service.h
@@ -34,7 +34,8 @@ public:
     enum Value : uint8_t
     {
         HTTP,
-        UNDEFINED
+        UNDEFINED,
+        MAX
     };
 
     ServiceType() = default;
@@ -48,10 +49,10 @@ public:
     {
         switch (v)
         {
-        case UNDEFINED:
-            return "(not set)";
         case HTTP:
             return "http";
+        case UNDEFINED: // fallthrough
+        case MAX:       // fallthrough
         default:
             return "(not set)";
         }
@@ -98,9 +99,10 @@ protected:
     const uint32_t tenant_id;
     std::vector<std::string> fields;
     std::vector<std::string> events;
-    ExtractorLogger* logger;
-    const ServiceBlueprint& sbp;
 
+    ExtractorLogger* logger = nullptr;
+
+    const ServiceBlueprint& sbp;
     const ServiceType type;
 };
 

diff --git a/src/network_inspectors/extractor/extractor_writer.cc b/src/network_inspectors/extractor/extractor_writer.cc
index cb35903939477625401df8c2412cf6aa4a95c847..11e5216742d2fe5af963b417c5cb509b59260094 100644 (file)
--- a/src/network_inspectors/extractor/extractor_writer.cc
+++ b/src/network_inspectors/extractor/extractor_writer.cc
@@ -17,6 +17,10 @@
 //--------------------------------------------------------------------------
 // extractor_writer.cc author Anna Norokh <anorokh@cisco.com>
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include "extractor_writer.h"
 
 ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type)
@@ -25,7 +29,8 @@ ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type)
     {
     case OutputType::STD:
         return new StdExtractorWriter();
+    case OutputType::MAX: // fallthrough
+    default:
+        return nullptr;
     }
-
-    return nullptr;
 }

diff --git a/src/network_inspectors/extractor/extractor_writer.h b/src/network_inspectors/extractor/extractor_writer.h
index 3173f0806bc0f581776d283680036dc76f89f0d1..d6551f5cf1f7f4aebb4ac12afc552c8e04f52b3e 100644 (file)
--- a/src/network_inspectors/extractor/extractor_writer.h
+++ b/src/network_inspectors/extractor/extractor_writer.h
@@ -28,7 +28,8 @@ class OutputType
 public:
     enum Value : uint8_t
     {
-        STD
+        STD,
+        MAX
     };
 
     OutputType() = default;
@@ -44,6 +45,7 @@ public:
         {
         case STD:
             return "stdout";
+        case MAX: // fallthrough
         default:
             return "(not set)";
         }