]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Pull request #4317: extractor: add protocol logging for HTTP
authorAnna Norokh -X (anorokh - SOFTSERVE INC at Cisco) <anorokh@cisco.com>
Thu, 13 Jun 2024 07:11:28 +0000 (07:11 +0000)
committerOleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) <oshumeik@cisco.com>
Thu, 13 Jun 2024 07:11:28 +0000 (07:11 +0000)
Merge in SNORT/snort3 from ~ANOROKH/snort3:extractor to master

Squashed commit of the following:

commit 2139770ad6bffa4e27f5f1ec4ca76dfcd950588d
Author: Anna Norokh <anorokh@cisco.com>
Date:   Fri Apr 5 13:46:28 2024 +0300

    extractor: add protocol logging for HTTP

    - added module and inspector implementation;
    - added logger and writer implementation for csv output to stdout;
    - added new pub/sub for end of transaction event;

31 files changed:
src/network_inspectors/CMakeLists.txt
src/network_inspectors/extractor/CMakeLists.txt [new file with mode: 0644]
src/network_inspectors/extractor/dev_notes.txt [new file with mode: 0644]
src/network_inspectors/extractor/extractor.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor.h [new file with mode: 0644]
src/network_inspectors/extractor/extractor_csv_logger.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor_csv_logger.h [new file with mode: 0644]
src/network_inspectors/extractor/extractor_event_handlers.h [new file with mode: 0644]
src/network_inspectors/extractor/extractor_http_event_handler.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor_logger.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor_logger.h [new file with mode: 0644]
src/network_inspectors/extractor/extractor_service.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor_service.h [new file with mode: 0644]
src/network_inspectors/extractor/extractor_writer.cc [new file with mode: 0644]
src/network_inspectors/extractor/extractor_writer.h [new file with mode: 0644]
src/network_inspectors/network_inspectors.cc
src/pub_sub/CMakeLists.txt
src/pub_sub/http_event_ids.h
src/pub_sub/http_transaction_end_event.cc [new file with mode: 0644]
src/pub_sub/http_transaction_end_event.h [new file with mode: 0644]
src/pub_sub/test/CMakeLists.txt
src/pub_sub/test/pub_sub_http_request_body_event_test.cc
src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc [new file with mode: 0644]
src/service_inspectors/http_inspect/http_enum.h
src/service_inspectors/http_inspect/http_msg_section.cc
src/service_inspectors/http_inspect/http_tables.cc
src/service_inspectors/http_inspect/http_transaction.cc
src/service_inspectors/http_inspect/http_transaction.h
src/service_inspectors/http_inspect/ips_http_version.cc
src/service_inspectors/http_inspect/test/http_transaction_test.cc
src/service_inspectors/http_inspect/test/http_unit_test_helpers.h [new file with mode: 0644]

index 0b7b881e6e865d1b4e5d06f60dc7c0da15a443ec..d6df6900f8ae1ec0251f12482f5982535a42b972 100644 (file)
@@ -2,6 +2,7 @@
 add_subdirectory(appid)
 add_subdirectory(arp_spoof)
 add_subdirectory(binder)
+add_subdirectory(extractor)
 add_subdirectory(kaizen)
 add_subdirectory(normalize)
 add_subdirectory(packet_capture)
@@ -21,6 +22,7 @@ endif()
 set(STATIC_NETWORK_INSPECTOR_PLUGINS
     $<TARGET_OBJECTS:appid>
     $<TARGET_OBJECTS:binder>
+    $<TARGET_OBJECTS:extractor>
     $<TARGET_OBJECTS:kaizen>
     $<TARGET_OBJECTS:normalize>
     $<TARGET_OBJECTS:port_scan>
diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt
new file mode 100644 (file)
index 0000000..f6451cc
--- /dev/null
@@ -0,0 +1,16 @@
+set( FILE_LIST
+    extractor.cc
+    extractor.h
+    extractor_csv_logger.cc
+    extractor_csv_logger.h
+    extractor_event_handlers.h
+    extractor_http_event_handler.cc
+    extractor_logger.cc
+    extractor_logger.h
+    extractor_service.cc
+    extractor_service.h
+    extractor_writer.cc
+    extractor_writer.h
+)
+
+add_library(extractor OBJECT ${FILE_LIST})
diff --git a/src/network_inspectors/extractor/dev_notes.txt b/src/network_inspectors/extractor/dev_notes.txt
new file mode 100644 (file)
index 0000000..a60ba92
--- /dev/null
@@ -0,0 +1,48 @@
+Extractor is a global network inspector that logs flow data upon receiving
+a flow event.
+
+Supported services:
+ * HTTP
+ * HTTP2
+
+Supported events:
+ * end of HTTP transaction (request-response pair)
+
+An example configuration follows:
+
+    extractor =
+    {
+        protocols =
+        {
+            service = 'http',
+            tenant_id = 1,
+            on_events = 'eot',
+            fields = 'ts, uri, host, method'
+        }
+        {
+            service = 'http',
+            tenant_id = 2,
+            on_events = 'eot',
+            fields = 'ts, uri'
+        }
+    }
+
+Each tenant can have its own protocol configuration.
+
+A list of common fields which are logged:
+ * ts (timestamp)
+ * id.orig_h (client IP address)
+ * id.orig_p (client TCP port)
+ * id.resp_h (server IP address)
+ * id.resp_p (server TCP port)
+ * pkt_num (packet number)
+
+The following fields are supported for HTTP:
+ * method
+ * host
+ * uri
+ * user_agent
+ * version
+ * status_code
+ * status_msg
+
diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc
new file mode 100644 (file)
index 0000000..1234b52
--- /dev/null
@@ -0,0 +1,270 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor.cc author Anna Norokh <anorokh@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "extractor.h"
+
+#include <algorithm>
+
+#include "framework/data_bus.h"
+#include "framework/inspector.h"
+#include "framework/module.h"
+#include "log/messages.h"
+#include "main/snort_config.h"
+#include "protocols/packet.h"
+#include "pub_sub/http_events.h"
+
+#include "extractor_event_handlers.h"
+#include "extractor_logger.h"
+#include "extractor_service.h"
+
+using namespace snort;
+
+THREAD_LOCAL ExtractorStats extractor_stats;
+THREAD_LOCAL ProfileStats extractor_perf_stats;
+
+//-------------------------------------------------------------------------
+// module stuff
+//-------------------------------------------------------------------------
+
+static const Parameter extractor_proto_params[] =
+{
+    { "service", Parameter::PT_ENUM, "http", nullptr,
+      "service to extract from" },
+
+    { "tenant_id", Parameter::PT_INT, "0:max32", "0",
+      "tenant_id of target tenant" },
+
+    { "on_events", Parameter::PT_STRING, nullptr, nullptr,
+      "specify events to log" },
+
+    { "fields", Parameter::PT_STRING, nullptr, nullptr,
+      "specify fields to log" },
+
+    { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
+};
+
+static const Parameter s_params[] =
+{
+    { "formatting", Parameter::PT_ENUM, "csv", "csv",
+      "output format for extractor" },
+
+    { "output", Parameter::PT_ENUM, "stdout", "stdout",
+      "output destination for extractor" },
+
+    { "protocols", Parameter::PT_LIST, extractor_proto_params, nullptr,
+      "protocols to extract data" },
+
+    { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
+};
+
+void ServiceConfig::clear()
+{
+    service = ServiceType::UNDEFINED;
+    on_events.clear();
+    tenant_id = 0;
+    fields.clear();
+}
+
+ExtractorModule::ExtractorModule() : Module(S_NAME, s_help, s_params) { }
+
+void ExtractorModule::commit_config()
+{
+    for (const auto& p : extractor_config.protocols)
+    {
+        if (p.tenant_id == service_config.tenant_id and p.service == service_config.service)
+        {
+            ParseError("%s service got multiple configurations", service_config.service.c_str());
+            break;
+        }
+    }
+
+    extractor_config.protocols.push_back(service_config);
+    service_config.clear();
+}
+
+static inline void trim(std::string& str)
+{
+    str.erase(str.find_last_not_of(' ') + 1);
+    str.erase(0, str.find_first_not_of(' '));
+}
+
+void ExtractorModule::store(Value& val, std::vector<std::string>& dst)
+{
+    dst.clear();
+    val.set_first_token();
+    std::string tok;
+    while (val.get_next_csv_token(tok))
+    {
+        trim(tok);
+        dst.push_back(tok);
+    }
+}
+
+bool ExtractorModule::begin(const char*, int idx, SnortConfig*)
+{
+    if (idx == 0)
+    {
+        service_config.clear();
+        extractor_config.protocols.clear();
+    }
+
+    return true;
+}
+
+bool ExtractorModule::set(const char*, Value& v, SnortConfig*)
+{
+    if (v.is("formatting"))
+        extractor_config.formatting = (FormatType)(v.get_uint8());
+
+    else if (v.is("output"))
+        extractor_config.output = (OutputType)(v.get_uint8());
+
+    else if (v.is("service"))
+        service_config.service = (ServiceType)(v.get_uint8());
+
+    else if (v.is("tenant_id"))
+        service_config.tenant_id = v.get_uint32();
+
+    else if (v.is("on_events"))
+        store(v, service_config.on_events);
+
+    else if (v.is("fields"))
+        store(v, service_config.fields);
+
+    return true;
+}
+
+bool ExtractorModule::end(const char* fqn, int idx, SnortConfig*)
+{
+    if (idx > 0 && !strcmp(fqn, "extractor.protocols"))
+    {
+        if (service_config.fields.empty())
+        {
+            ParseError("Can't initialize extractor without protocols.fields");
+            return false;
+        }
+        commit_config();
+    }
+
+    return true;
+}
+
+//-------------------------------------------------------------------------
+// Inspector stuff
+//-------------------------------------------------------------------------
+
+Extractor::Extractor(ExtractorModule* m)
+{
+    auto& cfg = m->get_config();
+
+    format = cfg.formatting;
+    output = cfg.output;
+
+    for (const auto& p : cfg.protocols)
+    {
+        auto s = ExtractorService::make_service(p, format, output);
+
+        if (s)
+            services.push_back(s);
+    }
+}
+
+Extractor::~Extractor()
+{
+    for (const auto& s : services)
+        delete s;
+}
+
+void Extractor::show(const SnortConfig*) const
+{
+    ConfigLogger::log_value("formatting", format.c_str());
+    ConfigLogger::log_value("output", output.c_str());
+
+    bool log_header = true;
+    for (const auto& s : services)
+    {
+        if (log_header)
+        {
+            ConfigLogger::log_option("protocols");
+            log_header = false;
+        }
+        std::string str;
+        s->show(str);
+
+        ConfigLogger::log_list("", str.c_str(), "   ");
+    }
+}
+//-------------------------------------------------------------------------
+// api stuff
+//-------------------------------------------------------------------------
+
+static Module* mod_ctor()
+{ return new ExtractorModule; }
+
+static void mod_dtor(Module* m)
+{ delete m; }
+
+static Inspector* extractor_ctor(Module* mod)
+{ return new Extractor((ExtractorModule*)mod); }
+
+static void extractor_dtor(Inspector* p)
+{ delete p; }
+
+static InspectApi extractor_api =
+{
+    {
+        PT_INSPECTOR,
+        sizeof(InspectApi),
+        INSAPI_VERSION,
+        0,
+        API_RESERVED,
+        API_OPTIONS,
+        S_NAME,
+        s_help,
+        mod_ctor,
+        mod_dtor
+    },
+    IT_PASSIVE,
+    PROTO_BIT__ANY_TYPE,
+    nullptr, // buffers
+    nullptr, // service
+    nullptr, // pinit
+    nullptr, // pterm
+    nullptr, // tinit
+    nullptr, // tterm
+    extractor_ctor,
+    extractor_dtor,
+    nullptr, // ssn
+    nullptr  // reset
+};
+
+#ifdef BUILDING_SO
+SO_PUBLIC const BaseApi* snort_plugins[] =
+#else
+const BaseApi* nin_extractor[] =
+#endif
+{
+    &extractor_api.base,
+    nullptr
+};
+
diff --git a/src/network_inspectors/extractor/extractor.h b/src/network_inspectors/extractor/extractor.h
new file mode 100644 (file)
index 0000000..f4f4920
--- /dev/null
@@ -0,0 +1,118 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor.h author Anna Norokh <anorokh@cisco.com>
+
+#ifndef EXTRACTOR_H
+#define EXTRACTOR_H
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "framework/inspector.h"
+#include "framework/module.h"
+#include "main/snort_config.h"
+#include "profiler/profiler.h"
+
+#include "extractor_logger.h"
+#include "extractor_service.h"
+#include "extractor_writer.h"
+
+#define S_NAME "extractor"
+#define s_help "extracts protocol specific data"
+
+class ServiceConfig
+{
+public:
+    ServiceConfig() : service(ServiceType::UNDEFINED), tenant_id(0) {}
+    void clear();
+
+    ServiceType service;
+    uint32_t tenant_id;
+    std::vector<std::string> on_events;
+    std::vector<std::string> fields;
+};
+
+struct ExtractorConfig
+{
+    FormatType formatting = FormatType::CSV;
+    OutputType output = OutputType::STD;
+    std::vector<ServiceConfig> protocols;
+};
+
+static const PegInfo extractor_pegs[] =
+{
+    { CountType::SUM, "total_events", "total extractor events" },
+    { CountType::END, nullptr, nullptr }
+};
+
+struct ExtractorStats
+{
+    PegCount total_event;
+};
+
+extern THREAD_LOCAL ExtractorStats extractor_stats;
+extern THREAD_LOCAL snort::ProfileStats extractor_perf_stats;
+
+class ExtractorModule : public snort::Module
+{
+public:
+    ExtractorModule();
+
+    const PegInfo* get_pegs() const override
+    { return extractor_pegs; }
+
+    PegCount* get_counts() const override
+    { return (PegCount*)&extractor_stats; }
+
+    snort::ProfileStats* get_profile() const override
+    { return &extractor_perf_stats; }
+
+    bool begin(const char*, int, snort::SnortConfig*) override;
+    bool set(const char*, snort::Value& v, snort::SnortConfig*) override;
+    bool end(const char*, int, snort::SnortConfig*) override;
+
+    Usage get_usage() const override
+    { return GLOBAL; }
+
+    const ExtractorConfig& get_config()
+    { return extractor_config; }
+
+private:
+    void store(snort::Value& val, std::vector<std::string>& dst);
+    void commit_config();
+
+    ExtractorConfig extractor_config;
+    ServiceConfig service_config;
+};
+
+class Extractor : public snort::Inspector
+{
+public:
+    Extractor(ExtractorModule*);
+    ~Extractor() override;
+
+    void show(const snort::SnortConfig*) const override;
+
+private:
+    std::vector<ExtractorService*> services;
+    FormatType format;
+    OutputType output;
+};
+
+#endif
diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc
new file mode 100644 (file)
index 0000000..9180020
--- /dev/null
@@ -0,0 +1,85 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// csv_logger.cc author Anna Norokh <anorokh@cisco.com>
+
+#include "extractor_csv_logger.h"
+
+#include <algorithm>
+#include <cassert>
+
+THREAD_LOCAL bool first_write;
+
+void CsvExtractorLogger::add_header()
+{
+    std::string header;
+
+    header += "#";
+    header += fields_name[0];
+    for (size_t i = 1; i < fields_name.size(); ++i)
+    {
+        header += ",";
+        header += fields_name[i];
+    }
+    header += "\n";
+
+    writer->write(header.c_str());
+}
+
+void CsvExtractorLogger::open_record()
+{
+    first_write = true;
+    writer->lock();
+}
+
+void CsvExtractorLogger::close_record()
+{
+    writer->write("\n");
+    writer->unlock();
+}
+
+void CsvExtractorLogger::add_field(const snort::Value& v)
+{
+    switch (v.get_type())
+    {
+    case snort::Value::ValueType::VT_UNUM:
+    {
+        first_write ? []() { first_write = false; } () : writer->write(",");
+        writer->write(std::to_string(v.get_uint64()).c_str());
+        break;
+    }
+
+    case snort::Value::ValueType::VT_STR:
+    {
+        first_write ? []() { first_write = false; } () : writer->write(",");
+        writer->write(v.get_string());
+        break;
+    }
+
+    case snort::Value::ValueType::VT_BOOL: // fallthrough
+    case snort::Value::ValueType::VT_NUM:  // fallthrough
+    case snort::Value::ValueType::VT_REAL: // fallthrough
+    default:
+        assert(false);
+        break;
+    }
+}
+
+CsvExtractorLogger::~CsvExtractorLogger()
+{
+    delete writer;
+}
diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h
new file mode 100644 (file)
index 0000000..abb8152
--- /dev/null
@@ -0,0 +1,45 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// csv_logger.h author Anna Norokh <anorokh@cisco.com>
+
+#ifndef CSV_LOGGER_H
+#define CSV_LOGGER_H
+
+#include "framework/value.h"
+
+#include "extractor_logger.h"
+#include "extractor_writer.h"
+
+class CsvExtractorLogger : public ExtractorLogger
+{
+public:
+    CsvExtractorLogger(OutputType o_type, const std::vector<std::string>& fields)
+        : ExtractorLogger(fields), writer(ExtractorWriter::make_writer(o_type))
+    { CsvExtractorLogger::add_header(); }
+    ~CsvExtractorLogger() override;
+
+    void add_header() override;
+    void add_field(const snort::Value&) override;
+    void open_record() override;
+    void close_record() override;
+
+private:
+    ExtractorWriter* const writer;
+};
+
+#endif
diff --git a/src/network_inspectors/extractor/extractor_event_handlers.h b/src/network_inspectors/extractor/extractor_event_handlers.h
new file mode 100644 (file)
index 0000000..ff0b490
--- /dev/null
@@ -0,0 +1,52 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_event_handlers.h author Maya Dagon <mdagon@cisco.com>
+
+#ifndef EXTRACTOR_EVENT_HANDLERS_H
+#define EXTRACTOR_EVENT_HANDLERS_H
+
+#include "framework/data_bus.h"
+
+#include "extractor.h"
+#include "extractor_logger.h"
+
+namespace snort
+{
+
+class ExtractorEvent
+{
+protected:
+    ExtractorEvent(uint32_t tid, const std::vector<std::string>& flds, ExtractorLogger& l)
+        : tenant_id(tid), fields(flds), logger(l) {}
+
+    uint32_t tenant_id;
+    const std::vector<std::string> fields;
+    ExtractorLogger& logger;
+};
+
+class HttpExtractorEventHandler : public DataHandler, public ExtractorEvent
+{
+public:
+    HttpExtractorEventHandler(uint32_t tenant, const std::vector<std::string>& flds,
+        ExtractorLogger& l) : DataHandler(S_NAME), ExtractorEvent(tenant, flds, l) {}
+
+    void handle(DataEvent&, Flow*) override;
+};
+
+}
+#endif
diff --git a/src/network_inspectors/extractor/extractor_http_event_handler.cc b/src/network_inspectors/extractor/extractor_http_event_handler.cc
new file mode 100644 (file)
index 0000000..a133cfe
--- /dev/null
@@ -0,0 +1,200 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_http_event_handler.cc author Maya Dagon <mdagon@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "extractor_event_handlers.h"
+
+#include "detection/detection_engine.h"
+#include "framework/value.h"
+#include "profiler/profiler.h"
+#include "pub_sub/http_transaction_end_event.h"
+#include "service_inspectors/http_inspect/http_transaction.h"
+#include "sfip/sf_ip.h"
+#include "utils/util.h"
+#include "utils/util_net.h"
+
+using namespace snort;
+
+
+typedef Value* (*GetFunc) (DataEvent*, Packet*, Flow*);
+
+// HttpTransactionEnd specific
+Value* get_method(DataEvent*, Packet*, Flow*);
+Value* get_host(DataEvent*, Packet*, Flow*);
+Value* get_user_agent(DataEvent*, Packet*, Flow*);
+Value* get_uri(DataEvent*, Packet*, Flow*);
+Value* get_version(DataEvent*, Packet*, Flow*);
+Value* get_stat_code(DataEvent*, Packet*, Flow*);
+Value* get_stat_msg(DataEvent*, Packet*, Flow*);
+
+// Common
+Value* get_timestamp(DataEvent*, Packet*, Flow*);
+Value* get_ip_src(DataEvent*, Packet*, Flow*);
+Value* get_ip_dst(DataEvent*, Packet*, Flow*);
+Value* get_ip_src_port(DataEvent*, Packet*, Flow*);
+Value* get_ip_dst_port(DataEvent*, Packet*, Flow*);
+Value* get_pkt_num(DataEvent*, Packet*, Flow*);
+
+static void field_to_string(const Field& field, std::string& value)
+{
+    if (field.length() > 0)
+        value.assign((const char*)field.start(), field.length());
+}
+
+Value* get_method(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_method();
+    std::string str;
+    field_to_string(field, str);
+    return new Value(str.c_str());
+}
+
+Value* get_host(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_host_hdr();
+    std::string str;
+    field_to_string(field, str);
+    return new Value(str.c_str());
+}
+
+Value* get_user_agent(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_user_agent();
+    std::string str;
+    field_to_string(field, str);
+    return new Value(str.c_str());
+}
+
+Value* get_uri(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_uri();
+    std::string str;
+    field_to_string(field, str);
+    return new Value(str.c_str());
+}
+
+Value* get_version(DataEvent* event, Packet*, Flow*)
+{
+    HttpEnums::VersionId version = ((HttpTransactionEndEvent*)event)->get_version();
+    const auto& iter = HttpEnums::VersionEnumToStr.find(version);
+    if (iter != HttpEnums::VersionEnumToStr.end())
+        return new Value(iter->second);
+
+    return new Value("");
+}
+
+Value* get_stat_code(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_stat_code();
+    std::string str;
+    field_to_string(field, str);
+
+    return new Value((uint64_t)atoi(str.c_str()));
+}
+
+Value* get_stat_msg(DataEvent* event, Packet*, Flow*)
+{
+    const Field& field = ((HttpTransactionEndEvent*)event)->get_stat_msg();
+    std::string str;
+    field_to_string(field, str);
+    return new Value(str.c_str());
+}
+
+Value* get_timestamp(DataEvent*, Packet* p, Flow*)
+{
+    char u_sec[8];
+    SnortSnprintf(u_sec, sizeof(u_sec),".%06d",(unsigned)p->pkth->ts.tv_usec);
+    auto str = std::to_string(p->pkth->ts.tv_sec) + u_sec;
+
+    return new Value(str.c_str());
+}
+
+Value* get_ip_src(DataEvent*, Packet*, Flow* flow)
+{
+    InetBuf src;
+    const SfIp& flow_srcip = flow->flags.client_initiated ? flow->client_ip : flow->server_ip;
+    sfip_ntop(&flow_srcip, src, sizeof(src));
+    std::string str = src;
+    return new Value(str.c_str());
+}
+
+Value* get_ip_dst(DataEvent*, Packet*, Flow* flow)
+{
+    InetBuf dst;
+    const SfIp& flow_dstip = flow->flags.client_initiated ? flow->server_ip : flow->client_ip;
+    sfip_ntop(&flow_dstip, dst, sizeof(dst));
+    std::string str = dst;
+    return new Value(str.c_str());
+}
+
+Value* get_ip_src_port(DataEvent*, Packet*, Flow* flow)
+{
+    return new Value((uint64_t)flow->client_port);
+}
+
+Value* get_ip_dst_port(DataEvent*, Packet*, Flow* flow)
+{
+    return new Value((uint64_t)flow->server_port);
+}
+
+Value* get_pkt_num(DataEvent*, Packet* p, Flow*)
+{
+    return new Value(p->context->packet_number);
+}
+
+static std::map<std::string, GetFunc> event_getters =
+{
+    {"ts", get_timestamp},
+    {"id.orig_h", get_ip_src},
+    {"id.resp_h", get_ip_dst},
+    {"id.orig_p", get_ip_src_port},
+    {"id.resp_p", get_ip_dst_port},
+    {"pkt_num", get_pkt_num},
+    {"method", get_method},
+    {"host", get_host},
+    {"uri", get_uri},
+    {"user_agent", get_user_agent},
+    {"version", get_version},
+    {"status_code", get_stat_code},
+    {"status_msg", get_stat_msg},
+};
+
+void HttpExtractorEventHandler::handle(DataEvent& event, Flow* flow)
+{
+    // cppcheck-suppress unreadVariable
+    Profile profile(extractor_perf_stats);
+
+    if (tenant_id != flow->tenant)
+        return;
+
+    Packet* p = DetectionEngine::get_current_packet();
+
+    logger.open_record();
+    for (const auto& field : fields)
+    {
+        auto val = std::unique_ptr<Value>(event_getters[field](&event, p, flow));
+        logger.add_field(*val.get());
+    }
+    logger.close_record();
+
+    extractor_stats.total_event++;
+}
diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc
new file mode 100644 (file)
index 0000000..b49abfd
--- /dev/null
@@ -0,0 +1,34 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_logger.cc author Anna Norokh <anorokh@cisco.com>
+
+#include "extractor_logger.h"
+
+#include "extractor_csv_logger.h"
+
+ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type,
+    const std::vector<std::string>& fields)
+{
+    switch (f_type)
+    {
+    case FormatType::CSV:
+        return new CsvExtractorLogger(o_type, fields);
+    }
+
+    return nullptr;
+}
diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h
new file mode 100644 (file)
index 0000000..359a183
--- /dev/null
@@ -0,0 +1,85 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_logger.h author Anna Norokh <anorokh@cisco.com>
+
+#ifndef EXTRACTOR_LOGGER_H
+#define EXTRACTOR_LOGGER_H
+
+#include <string>
+#include <vector>
+
+#include "framework/value.h"
+
+#include "extractor_writer.h"
+
+class FormatType
+{
+public:
+    enum Value : uint8_t
+    {
+        CSV
+    };
+
+    FormatType() = default;
+    constexpr FormatType(Value a) : v(a) {}
+    template<typename T> constexpr FormatType(T a) : v((Value)a) {}
+
+    constexpr operator Value() const { return v; }
+    explicit operator bool() const = delete;
+
+    const char* c_str() const
+    {
+        switch (v)
+        {
+        case CSV:
+            return "csv";
+        default:
+            return "(not set)";
+        }
+    }
+
+private:
+    Value v = CSV;
+};
+
+class ExtractorLogger
+{
+public:
+    static ExtractorLogger* make_logger(FormatType, OutputType, const std::vector<std::string>&);
+
+    ExtractorLogger() = delete;
+    ExtractorLogger(const ExtractorLogger&) = delete;
+    ExtractorLogger& operator=(const ExtractorLogger&) = delete;
+    ExtractorLogger(ExtractorLogger&&) = delete;
+
+    virtual ~ExtractorLogger() = default;
+
+    virtual void add_header() {}
+    virtual void add_footer() {}
+    virtual void add_field(const snort::Value&) {}
+
+    virtual void open_record() {}
+    virtual void close_record() {}
+
+protected:
+    ExtractorLogger(const std::vector<std::string>& fns) : fields_name(fns) {}
+
+    const std::vector<std::string>& fields_name;
+};
+
+#endif
diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc
new file mode 100644 (file)
index 0000000..68ed292
--- /dev/null
@@ -0,0 +1,172 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_services.cc author Maya Dagon <mdagon@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "extractor_service.h"
+
+#include "framework/data_bus.h"
+#include "log/messages.h"
+#include "pub_sub/http_events.h"
+
+#include "extractor.h"
+#include "extractor_event_handlers.h"
+
+using namespace snort;
+
+
+//////////////////////////////////////////////////////////////////////
+////  ExtractorService
+//////////////////////////////////////////////////////////////////////
+
+std::vector<std::string> ExtractorService::common_fields =
+{
+    "ts",
+    "id.orig_h",
+    "id.orig_p",
+    "id.resp_h",
+    "id.resp_p",
+    "pkt_num"
+};
+
+
+ExtractorService::ExtractorService(uint32_t tenant, const std::vector<std::string>& srv_fields,
+    const std::vector<std::string>& srv_events, const ServiceBlueprint& srv_bp,
+    ServiceType s_type, FormatType f_type, OutputType o_type) : tenant_id(tenant), sbp(srv_bp), type(s_type)
+{
+    add_fields(srv_fields);
+    add_events(srv_events);
+
+    logger = ExtractorLogger::make_logger(f_type, o_type, get_fields());
+}
+
+void ExtractorService::add_events(const std::vector<std::string>& vals)
+{
+    for (const auto& val : vals)
+    {
+        if (find_event(val))
+            events.push_back(val);
+        else
+            ParseError("Invalid protocols.on_events value %s", val.c_str());
+    }
+}
+
+void ExtractorService::add_fields(const std::vector<std::string>& vals)
+{
+    for (auto& val : vals)
+    {
+        if (find_field(val))
+            fields.push_back(val);
+        else
+            ParseError("Invalid protocols.fields value %s", val.c_str());
+    }
+}
+
+ExtractorService* ExtractorService::make_service(const ServiceConfig& cfg, FormatType f_type, OutputType o_type)
+{
+    if (cfg.on_events.empty())
+        ParseError("%s service misses on_events field", cfg.service.c_str());
+
+    switch (cfg.service)
+    {
+        case ServiceType::HTTP:
+            return new HttpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, f_type, o_type);
+
+        case ServiceType::UNDEFINED:
+            ParseError("%s service is not supported", cfg.service.c_str());
+            break;
+
+        default:
+            return nullptr;
+    }
+    return nullptr;
+}
+
+bool ExtractorService::find_event(const std::string& event) const
+{
+    return std::find(sbp.supported_events.begin(), sbp.supported_events.end(), event)
+        != sbp.supported_events.end();
+}
+
+bool ExtractorService::find_field(const std::string& field) const
+{
+    return ((std::find(common_fields.begin(), common_fields.end(), field) != common_fields.end()) or
+        (std::find(sbp.supported_fields.begin(), sbp.supported_fields.end(),field)
+          != sbp.supported_fields.end()));
+}
+
+void ExtractorService::show(std::string& str) const
+{
+    str = "{ service = ";
+    str += type.c_str();
+    str += ", tenant_id = ";
+    str += std::to_string(tenant_id);
+    str += ", on_events =";
+    for (const auto& event : get_events())
+    {
+        str += " ";
+        str += event;
+    }
+    str += ", fields = ";
+    for (const auto& field : get_fields())
+    {
+        str += field;
+        str += " ";
+    }
+    str += " }";
+}
+
+//////////////////////////////////////////////////////////////////////
+////  HttpExtractorService
+//////////////////////////////////////////////////////////////////////
+
+ServiceBlueprint HttpExtractorService::blueprint =
+{
+    // events
+    {
+      "eot",
+    },
+    // fields
+    {
+      "method",
+      "host",
+      "uri",
+      "user_agent",
+      "version",
+      "status_code",
+      "status_msg",
+    },
+};
+
+HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector<std::string>& srv_fields,
+    const std::vector<std::string>& srv_events, ServiceType s_type, FormatType f_type, OutputType o_type)
+    : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, f_type, o_type)
+{
+    for (const auto& event : get_events())
+    {
+        if (!strcmp("eot", event.c_str()))
+        {
+            DataBus::subscribe(http_pub_key, HttpEventIds::END_OF_TRANSACTION,
+                new HttpExtractorEventHandler(tenant_id, get_fields(), *logger));
+        }
+    }
+}
+
diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h
new file mode 100644 (file)
index 0000000..0afeb12
--- /dev/null
@@ -0,0 +1,118 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_service.h author Maya Dagon <mdagon@cisco.com>
+
+#ifndef EXTRACTOR_SERVICES_H
+#define EXTRACTOR_SERVICES_H
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "extractor_logger.h"
+
+class ServiceConfig;
+
+class ServiceType
+{
+public:
+    enum Value : uint8_t
+    {
+        HTTP,
+        UNDEFINED
+    };
+
+    ServiceType() = default;
+    constexpr ServiceType(Value a) : v(a) {}
+    template<typename T> constexpr ServiceType(T a) : v(Value(a)) {}
+
+    constexpr operator Value() const { return v; }
+    explicit operator bool() const = delete;
+
+    const char* c_str() const
+    {
+        switch (v)
+        {
+        case UNDEFINED:
+            return "(not set)";
+        case HTTP:
+            return "http";
+        default:
+            return "(not set)";
+        }
+    }
+
+private:
+    Value v = UNDEFINED;
+};
+
+struct ServiceBlueprint
+{
+    std::vector<std::string> supported_events;
+    std::vector<std::string> supported_fields;
+};
+
+class ExtractorService
+{
+public:
+    static ExtractorService* make_service(const ServiceConfig&, FormatType, OutputType);
+
+    ExtractorService() = delete;
+    ExtractorService(const ExtractorService&) = delete;
+    ExtractorService& operator=(const ExtractorService&) = delete;
+    ExtractorService(ExtractorService&&) = delete;
+
+    virtual ~ExtractorService()
+    { delete logger; }
+
+    void show(std::string&) const;
+    uint32_t get_tenant() const { return tenant_id; }
+    const std::vector<std::string>& get_events() const { return events; }
+    const std::vector<std::string>& get_fields() const { return fields; }
+
+protected:
+    ExtractorService(uint32_t tenant, const std::vector<std::string>& fields, const std::vector<std::string>& events,
+        const ServiceBlueprint& srv_bp, ServiceType, FormatType, OutputType);
+    void add_events(const std::vector<std::string>& vals);
+    void add_fields(const std::vector<std::string>& vals);
+    bool find_event(const std::string&) const;
+    bool find_field(const std::string&) const;
+
+    static std::vector<std::string> common_fields;
+
+    const uint32_t tenant_id;
+    std::vector<std::string> fields;
+    std::vector<std::string> events;
+    ExtractorLogger* logger;
+    const ServiceBlueprint& sbp;
+
+    const ServiceType type;
+};
+
+class HttpExtractorService : public ExtractorService
+{
+public:
+    HttpExtractorService(uint32_t tenant, const std::vector<std::string>& fields,
+    const std::vector<std::string>& events, ServiceType, FormatType, OutputType);
+
+private:
+    static ServiceBlueprint blueprint;
+};
+
+#endif
+
diff --git a/src/network_inspectors/extractor/extractor_writer.cc b/src/network_inspectors/extractor/extractor_writer.cc
new file mode 100644 (file)
index 0000000..cb35903
--- /dev/null
@@ -0,0 +1,31 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_writer.cc author Anna Norokh <anorokh@cisco.com>
+
+#include "extractor_writer.h"
+
+ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type)
+{
+    switch (o_type)
+    {
+    case OutputType::STD:
+        return new StdExtractorWriter();
+    }
+
+    return nullptr;
+}
diff --git a/src/network_inspectors/extractor/extractor_writer.h b/src/network_inspectors/extractor/extractor_writer.h
new file mode 100644 (file)
index 0000000..3173f08
--- /dev/null
@@ -0,0 +1,93 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// extractor_writer.h author Anna Norokh <anorokh@cisco.com>
+
+#ifndef EXTRACTOR_WRITER_H
+#define EXTRACTOR_WRITER_H
+
+#include <mutex>
+#include <string>
+
+class OutputType
+{
+public:
+    enum Value : uint8_t
+    {
+        STD
+    };
+
+    OutputType() = default;
+    constexpr OutputType(Value a) : v(a) {}
+    template<typename T> constexpr OutputType(T a) : v((Value)a) {}
+
+    constexpr operator Value() const { return v; }
+    explicit operator bool() const = delete;
+
+    const char* c_str() const
+    {
+        switch (v)
+        {
+        case STD:
+            return "stdout";
+        default:
+            return "(not set)";
+        }
+    }
+
+private:
+    Value v = STD;
+};
+
+class ExtractorWriter
+{
+public:
+    static ExtractorWriter* make_writer(OutputType);
+
+    ExtractorWriter(const ExtractorWriter&) = delete;
+    ExtractorWriter& operator=(const ExtractorWriter&) = delete;
+    ExtractorWriter(ExtractorWriter&&) = delete;
+
+    virtual ~ExtractorWriter() = default;
+
+    virtual void write(const char*) = 0;
+    virtual void lock() { }
+    virtual void unlock() { }
+
+protected:
+    ExtractorWriter() = default;
+};
+
+class StdExtractorWriter : public ExtractorWriter
+{
+public:
+    StdExtractorWriter() = default;
+
+    void write(const char* ss) override
+    { fprintf(stdout, "%s", ss); }
+
+    void lock() override
+    { write_mutex.lock(); }
+
+    void unlock() override
+    { write_mutex.unlock(); }
+
+private:
+    std::mutex write_mutex;
+};
+
+#endif
index 399f1fd18877bc5618657fdf0be4aca773b3a368..4b7cd53e0176aee75c903a1dd0100993c1d94318 100644 (file)
@@ -31,6 +31,7 @@ extern const BaseApi* nin_normalize;
 extern const BaseApi* nin_reputation;
 
 extern const BaseApi* nin_appid[];
+extern const BaseApi* nin_extractor[];
 extern const BaseApi* nin_kaizen_engine[];
 extern const BaseApi* nin_kaizen[];
 extern const BaseApi* nin_port_scan[];
@@ -54,6 +55,7 @@ void load_network_inspectors()
 {
     PluginManager::load_plugins(network_inspectors);
     PluginManager::load_plugins(nin_appid);
+    PluginManager::load_plugins(nin_extractor);
     PluginManager::load_plugins(nin_kaizen_engine);
     PluginManager::load_plugins(nin_kaizen);
     PluginManager::load_plugins(nin_port_scan);
index c8af9e779adc7faaaed0e769329fb529b8f83523..2306be5148a313ee707fb293b853863914ce3e7a 100644 (file)
@@ -15,6 +15,7 @@ set (PUB_SUB_INCLUDES
     http_event_ids.h
     http_events.h
     http_request_body_event.h
+    http_transaction_end_event.h
     intrinsic_event_ids.h
     netflow_event.h
     opportunistic_tls_event.h
@@ -35,6 +36,7 @@ add_library( pub_sub OBJECT
     http_events.cc
     dns_events.cc
     http_request_body_event.cc
+    http_transaction_end_event.cc
     sip_events.cc
 )
 
index 72069f7c6831f13cf3162656289d25c9693b5966..30d2335f0e04dd0e25b8a71d8e6f6f571061234b 100644 (file)
@@ -35,6 +35,7 @@ struct HttpEventIds
     REQUEST_HEADER,
     RESPONSE_HEADER,
     REQUEST_BODY,
+    END_OF_TRANSACTION,  
 
     num_ids
 }; };
diff --git a/src/pub_sub/http_transaction_end_event.cc b/src/pub_sub/http_transaction_end_event.cc
new file mode 100644 (file)
index 0000000..78c9727
--- /dev/null
@@ -0,0 +1,94 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_transaction_end_event.cc author Maya Dagon <mdagon@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "http_transaction_end_event.h"
+
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_msg_header.h"
+#include "service_inspectors/http_inspect/http_msg_request.h"
+#include "service_inspectors/http_inspect/http_msg_section.h"
+#include "service_inspectors/http_inspect/http_msg_status.h"
+#include "service_inspectors/http_inspect/http_transaction.h"
+
+using namespace snort;
+
+HttpTransactionEndEvent::HttpTransactionEndEvent(const HttpTransaction* const trans)
+    : transaction(trans) { }
+
+const Field& HttpTransactionEndEvent::get_host_hdr() const
+{
+    HttpMsgHeader* headers = transaction->get_header(HttpCommon::SRC_CLIENT);
+    if (headers == nullptr)
+        return Field::FIELD_NULL;
+
+    return headers->get_classic_buffer(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_HOST, 0);
+}
+
+const Field& HttpTransactionEndEvent::get_uri() const
+{
+    if (transaction->get_request() == nullptr)
+        return Field::FIELD_NULL;
+
+    return transaction->get_request()->get_classic_buffer(HttpEnums::HTTP_BUFFER_URI, 0, 0);
+}
+
+const Field& HttpTransactionEndEvent::get_method() const
+{
+    if (transaction->get_request() == nullptr)
+        return Field::FIELD_NULL;
+
+    return transaction->get_request()->get_method();
+}
+
+const Field& HttpTransactionEndEvent::get_stat_code() const
+{
+    if (transaction->get_status() == nullptr)
+        return Field::FIELD_NULL;
+
+    return transaction->get_status()->get_status_code();
+}
+
+const Field& HttpTransactionEndEvent::get_stat_msg() const
+{
+    if (transaction->get_status() == nullptr)
+        return Field::FIELD_NULL;
+
+    return transaction->get_status()->get_reason_phrase();
+}
+
+const Field& HttpTransactionEndEvent::get_user_agent() const
+{
+    HttpMsgHeader* headers = transaction->get_header(HttpCommon::SRC_CLIENT);
+    if (headers == nullptr)
+        return Field::FIELD_NULL;
+
+    return headers->get_classic_buffer(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_USER_AGENT, 0);
+}
+
+HttpEnums::VersionId HttpTransactionEndEvent::get_version() const
+{
+    auto status = transaction->get_status();
+    if (!status and !transaction->get_request())
+        return HttpEnums::VERS__NOT_PRESENT;
+    return status ? status->get_version_id() : transaction->get_request()->get_version_id();
+}
diff --git a/src/pub_sub/http_transaction_end_event.h b/src/pub_sub/http_transaction_end_event.h
new file mode 100644 (file)
index 0000000..7cd4a05
--- /dev/null
@@ -0,0 +1,54 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_transaction_end_event.h author Maya Dagon <mdagon@cisco.com>
+
+#ifndef HTTP_TRANSACTION_END_EVENT_H
+#define HTTP_TRANSACTION_END_EVENT_H
+
+#include "framework/data_bus.h"
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_field.h"
+
+#include "http_event_ids.h"
+
+class HttpFlowData;
+class HttpMsgRequest;
+class HttpMsgStatus;
+class HttpTransaction;
+
+namespace snort
+{
+// This event is published each time a transaction is ending
+class SO_PUBLIC HttpTransactionEndEvent : public snort::DataEvent
+{
+public:
+    HttpTransactionEndEvent(const HttpTransaction* const);
+
+    const Field& get_host_hdr() const;
+    const Field& get_uri() const;
+    const Field& get_method() const;
+    const Field& get_stat_code() const;
+    const Field& get_stat_msg() const;
+    const Field& get_user_agent() const;
+    HttpEnums::VersionId get_version() const;
+
+private:
+    const HttpTransaction* const transaction;
+};
+}
+#endif
index 0a8892ae9ca8a6b05e58689c935bead4dc949e50..568ba067a7c8cecb136e36ceed3a6886b32775c1 100644 (file)
@@ -11,3 +11,12 @@ add_cpputest( pub_sub_eve_process_event_test
     SOURCES
         ../eve_process_event.h
 )
+add_cpputest( pub_sub_http_transaction_end_event_test
+    SOURCES
+        ../http_transaction_end_event.cc
+        ../../service_inspectors/http_inspect/http_transaction.cc
+        ../../service_inspectors/http_inspect/http_flow_data.cc
+        ../../service_inspectors/http_inspect/http_test_manager.cc
+        ../../service_inspectors/http_inspect/http_test_input.cc
+    LIBS ${ZLIB_LIBRARIES}
+)
index 34f7a53ffa1e50b3c40d956e819da72a1a7e6d1c..70c316c50421ab108b9afc542a5eb84ebe8da1cb 100644 (file)
@@ -72,7 +72,7 @@ HttpMsgSection::HttpMsgSection(const uint8_t* buffer, const uint16_t buf_size,
     session_data(session_data_),
     flow(flow_),
     params(params_),
-    transaction(HttpTransaction::attach_my_transaction(session_data, source_id_)),
+    transaction(HttpTransaction::attach_my_transaction(session_data, source_id_, flow)),
     trans_num(STAT_NOT_PRESENT),
     status_code_num(STAT_NOT_PRESENT),
     source_id(source_id_),
@@ -88,7 +88,7 @@ HttpMsgSection::HttpMsgSection(const uint8_t* buffer, const uint16_t buf_size,
 void HttpMsgSection::update_depth() const{}
 bool HttpMsgSection::run_detection(snort::Packet*) { return true; }
 
-HttpTransaction*HttpTransaction::attach_my_transaction(HttpFlowData*, HttpCommon::SourceId)
+HttpTransaction*HttpTransaction::attach_my_transaction(HttpFlowData*, HttpCommon::SourceId, snort::Flow*)
     { return nullptr; }
 Field::Field(int32_t length, const uint8_t* start, bool own_the_buffer_) :
     strt(start), len(length), own_the_buffer(own_the_buffer_)
diff --git a/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc b/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc
new file mode 100644 (file)
index 0000000..0ba37bd
--- /dev/null
@@ -0,0 +1,143 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// pub_sub_http_transaction_end_event_test.cc author Maya Dagon <mdagon@cisco.com>
+// Unit test for the HttpTransactionEndEvent
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "pub_sub/http_transaction_end_event.h"
+#include "service_inspectors/http_inspect/http_common.h"
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_flow_data.h"
+#include "service_inspectors/http_inspect/http_inspect.h"
+#include "service_inspectors/http_inspect/http_module.h"
+#include "service_inspectors/http_inspect/http_msg_section.h"
+#include "service_inspectors/http_inspect/http_transaction.h"
+#include "service_inspectors/http_inspect/test/http_unit_test_helpers.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+#include <CppUTestExt/MockSupport.h>
+
+using namespace snort;
+using namespace HttpCommon;
+using namespace HttpEnums;
+
+namespace snort
+{
+unsigned FlowData::flow_data_id = 0;
+FlowData::FlowData(unsigned, Inspector*) : next(nullptr), prev(nullptr), handler(nullptr), id(0)
+{ }
+FlowData::~FlowData() = default;
+FlowData* Flow::get_flow_data(uint32_t) const { return nullptr; }
+int Flow::set_flow_data(FlowData*) { return 0; }
+Flow::~Flow() = default;
+unsigned DataBus::get_id(PubKey const&) { return 0; }
+void DataBus::publish(unsigned int, unsigned int, DataEvent&, Flow*) { }
+int DetectionEngine::queue_event(unsigned int, unsigned int) { return 0; }
+fd_status_t File_Decomp_StopFree(fd_session_t*) { return File_Decomp_OK; }
+Inspector::Inspector() { }
+Inspector::~Inspector() = default;
+bool Inspector::likes(Packet*) { return true; }
+bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return false; }
+class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
+const StreamBuffer StreamSplitter::reassemble(snort::Flow*, unsigned int, unsigned int, unsigned char const*, unsigned
+    int, unsigned int, unsigned int&)
+{
+    StreamBuffer buf { nullptr, 0 };
+    return buf;
+}
+unsigned StreamSplitter::max(snort::Flow*) { return 0; }
+}
+
+HttpParaList::UriParam::UriParam() { }
+HttpParaList::JsNormParam::~JsNormParam() { }
+HttpParaList::~HttpParaList() { }
+const Field Field::FIELD_NULL { STAT_NO_SOURCE };
+const Field& HttpMsgSection::get_classic_buffer(unsigned, uint64_t, uint64_t)
+{ return Field::FIELD_NULL; }
+HttpInspect::HttpInspect(const HttpParaList* para) :
+    params(para), xtra_trueip_id(0), xtra_uri_id(0),
+    xtra_host_id(0), xtra_jsnorm_id(0)
+{ }
+HttpInspect::~HttpInspect() = default;
+bool HttpInspect::configure(SnortConfig*) { return true; }
+void HttpInspect::show(const SnortConfig*) const { }
+bool HttpInspect::get_buf(unsigned, snort::Packet*, snort::InspectionBuffer&) { return true; }
+HttpCommon::SectionType HttpInspect::get_type_expected(snort::Flow*, HttpCommon::SourceId) const
+{ return SEC_DISCARD; }
+void HttpInspect::finish_hx_body(snort::Flow*, HttpCommon::SourceId, HttpCommon::HXBodyState,
+    bool) const { }
+void HttpInspect::set_hx_body_state(snort::Flow*, HttpCommon::SourceId, HttpCommon::HXBodyState) const { }
+bool HttpInspect::get_fp_buf(snort::InspectionBuffer::Type, snort::Packet*,
+    snort::InspectionBuffer&) { return false; }
+void HttpInspect::eval(snort::Packet*) { }
+void HttpInspect::eval(snort::Packet*, HttpCommon::SourceId, const uint8_t*, uint16_t) { }
+void HttpInspect::clear(snort::Packet*) { }
+bool HttpInspect::get_buf(snort::InspectionBuffer::Type, snort::Packet*, snort::InspectionBuffer&) { return false; }
+const uint8_t* HttpInspect::adjust_log_packet(snort::Packet*, uint16_t&) { return nullptr; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Packet*, const uint8_t*, uint32_t, uint32_t, uint32_t*)
+{ return StreamSplitter::FLUSH; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Flow*, const uint8_t*, uint32_t, uint32_t*)
+{ return StreamSplitter::FLUSH; }
+const snort::StreamBuffer HttpStreamSplitter::reassemble(snort::Flow*, unsigned, unsigned, const
+    uint8_t*, unsigned, uint32_t, unsigned&)
+{
+    StreamBuffer buf { nullptr, 0 };
+    return buf;
+}
+bool HttpStreamSplitter::finish(snort::Flow*) { return false; }
+void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t) { }
+
+THREAD_LOCAL PegCount HttpModule::peg_counts[PEG_COUNT_MAX] = { };
+
+TEST_GROUP(pub_sub_http_transaction_end_event_test)
+{
+    Flow* const flow = new Flow;
+    HttpParaList params;
+    HttpFlowData* flow_data = new HttpFlowData(flow, &params);
+    SectionType* const section_type = HttpUnitTestSetup::get_section_type(flow_data);
+    void setup() override
+    {
+        flow->gadget = new HttpInspect(&params);
+    }
+
+    void teardown() override
+    {
+        delete flow_data;
+        delete flow->gadget;
+        delete flow;
+    }
+};
+
+TEST(pub_sub_http_transaction_end_event_test, version_no_req_no_status)
+{
+    section_type[SRC_CLIENT] = SEC_REQUEST;
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
+    HttpTransactionEndEvent event(trans);
+    HttpEnums::VersionId version = event.get_version();
+    CHECK(version == HttpEnums::VERS__NOT_PRESENT);
+}
+
+int main(int argc, char** argv)
+{
+    return CommandLineTestRunner::RunAllTests(argc, argv);
+}
+
index 73a3fae1f3953f748b0c5c1453d28619c28a84ba..457d80e018fd1295ccc6b661f8fec51e9d93f809 100755 (executable)
@@ -21,6 +21,8 @@
 #define HTTP_ENUM_H
 
 #include <cstdint>
+#include <map>
+#include <string>
 
 namespace HttpEnums
 {
@@ -77,6 +79,7 @@ enum ChunkState { CHUNK_NEWLINES, CHUNK_ZEROS, CHUNK_LEADING_WS, CHUNK_NUMBER, C
     CHUNK_OPTIONS, CHUNK_HCRLF, CHUNK_DATA, CHUNK_DCRLF1, CHUNK_DCRLF2, CHUNK_BAD };
 
 // List of possible HTTP versions.
+// When making changes to this enum, also update VersionStrToEnum and VersionEnumToStr 
 enum VersionId { VERS__PROBLEMATIC=-12, VERS__NOT_PRESENT=-11, VERS__OTHER=1,
     VERS_1_0, VERS_1_1, VERS_2_0, VERS_3_0, VERS_0_9, VERS__MIN = VERS__PROBLEMATIC,
     VERS__MAX = VERS_0_9};
@@ -453,6 +456,8 @@ extern const bool is_sp_tab_cr_lf_vt_ff[256];
 extern const bool is_sp_tab_quote_dquote[256];
 extern const bool is_print_char[256]; // printable includes SP, tab, CR, LF
 extern const bool is_sp_comma[256];
+extern const std::map <std::string, VersionId> VersionStrToEnum;
+extern const std::map <HttpEnums::VersionId, const char*> VersionEnumToStr;
 
 } // end namespace HttpEnums
 
index c93e03db75ba62c0d5566d1975fab4800cfc13e6..91de737305df5b135456055016116747699cc6b4 100644 (file)
@@ -53,7 +53,7 @@ HttpMsgSection::HttpMsgSection(const uint8_t* buffer, const uint16_t buf_size,
     session_data(session_data_),
     flow(flow_),
     params(params_),
-    transaction(HttpTransaction::attach_my_transaction(session_data, source_id_)),
+    transaction(HttpTransaction::attach_my_transaction(session_data, source_id_, flow)),
     trans_num(session_data->expected_trans_num[source_id_]),
     status_code_num((source_id_ == SRC_SERVER) ? session_data->status_code_num : STAT_NOT_PRESENT),
     source_id(source_id_),
index 1fa24fa4c26ed9b81d115d2c3dc96e1e4275029f..be7827e933115d855e0e0fb802ff003a2bbe475e 100755 (executable)
@@ -689,3 +689,22 @@ const bool HttpEnums::is_print_char[256] =
     false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false
 };
 
+const std::map <std::string, VersionId> HttpEnums::VersionStrToEnum =
+{
+    { "malformed", VERS__PROBLEMATIC },
+    { "other", VERS__OTHER },
+    { "1.0", VERS_1_0 },
+    { "1.1", VERS_1_1 },
+    { "2.0", VERS_2_0 },
+    { "3.0", VERS_3_0 },
+    { "0.9", VERS_0_9 }
+};
+
+const std::map <VersionId, const char*> HttpEnums::VersionEnumToStr =
+{
+    { VERS_1_0, "1.0" },
+    { VERS_1_1, "1.1" },
+    { VERS_2_0, "2.0" },
+    { VERS_3_0, "3.0" },
+    { VERS_0_9, "0.9" }
+};
index d1a456e96d5cb1e94f423dd8953b54c2512dece9..29743754809256c1c07b39039ef48534ca615dbb 100644 (file)
 
 #include "http_transaction.h"
 
+#include "pub_sub/http_transaction_end_event.h"
+
 #include "http_common.h"
 #include "http_enum.h"
 #include "http_event.h"
+#include "http_inspect.h"
 #include "http_msg_body.h"
 #include "http_msg_header.h"
 #include "http_msg_request.h"
@@ -50,14 +53,24 @@ static void delete_section_list(HttpMsgSection* section_list)
     }
 }
 
-HttpTransaction::HttpTransaction(HttpFlowData* session_data_): session_data(session_data_)
+HttpTransaction::HttpTransaction(HttpFlowData* session_data_, snort::Flow* const f): session_data(session_data_), flow(f)
 {
     infractions[0] = nullptr;
     infractions[1] = nullptr;
+    HttpInspect* const hi = (session_data->is_for_httpx()) ?
+        (HttpInspect*)(flow->assistant_gadget) : (HttpInspect*)(flow->gadget);
+    pub_id = hi->get_pub_id();
+}
+
+void HttpTransaction::publish_end_of_transaction()
+{
+    HttpTransactionEndEvent http_event(this);
+    DataBus::publish(pub_id, HttpEventIds::END_OF_TRANSACTION, http_event, flow);
 }
 
 HttpTransaction::~HttpTransaction()
 {
+    publish_end_of_transaction();
     delete request;
     delete status;
     for (int k = 0; k <= 1; k++)
@@ -71,7 +84,7 @@ HttpTransaction::~HttpTransaction()
 }
 
 HttpTransaction* HttpTransaction::attach_my_transaction(HttpFlowData* session_data, SourceId
-    source_id)
+    source_id, Flow* const f)
 {
     // This factory method:
     // 1. creates new transactions for all request messages and orphaned response messages
@@ -126,7 +139,7 @@ HttpTransaction* HttpTransaction::attach_my_transaction(HttpFlowData* session_da
                 }
             }
         }
-        session_data->transaction[SRC_CLIENT] = new HttpTransaction(session_data);
+        session_data->transaction[SRC_CLIENT] = new HttpTransaction(session_data, f);
 
         // The StreamSplitter generates infractions related to this transaction while splitting the
         // request line and keeps them in temporary storage in the FlowData. Now we move them here.
@@ -160,7 +173,7 @@ HttpTransaction* HttpTransaction::attach_my_transaction(HttpFlowData* session_da
         if (session_data->pipeline_underflow)
         {
             // A previous underflow separated the two sides forever
-            session_data->transaction[SRC_SERVER] = new HttpTransaction(session_data);
+            session_data->transaction[SRC_SERVER] = new HttpTransaction(session_data, f);
         }
         else if ((session_data->transaction[SRC_SERVER] = session_data->take_from_pipeline()) ==
             nullptr)
@@ -171,7 +184,7 @@ HttpTransaction* HttpTransaction::attach_my_transaction(HttpFlowData* session_da
                 // Either there is no request at all or there is a request but a previous response
                 // already took it. Either way we have more responses than requests.
                 session_data->pipeline_underflow = true;
-                session_data->transaction[SRC_SERVER] = new HttpTransaction(session_data);
+                session_data->transaction[SRC_SERVER] = new HttpTransaction(session_data, f);
             }
 
             else if (session_data->type_expected[SRC_CLIENT] == SEC_REQUEST)
index f6c137a3e7f2dbc805d5d3a4a5faca0d63757485..c3981da211a21765cf32b34daf29ec39b1de79ce 100644 (file)
@@ -37,9 +37,9 @@ class HttpTransaction
 {
 public:
     ~HttpTransaction();
-    static HttpTransaction* attach_my_transaction(HttpFlowData* session_data,
-        HttpCommon::SourceId source_id);
-    static void delete_transaction(HttpTransaction* transaction, HttpFlowData* session_data);
+    static HttpTransaction* attach_my_transaction(HttpFlowData*,
+        HttpCommon::SourceId, snort::Flow* const);
+    static void delete_transaction(HttpTransaction*, HttpFlowData*);
 
     HttpMsgRequest* get_request() const { return request; }
     void set_request(HttpMsgRequest* request_) { request = request_; }
@@ -57,7 +57,7 @@ public:
         { trailer[source_id] = trailer_; }
     void set_body(HttpMsgBody* latest_body);
 
-    HttpInfractions* get_infractions(HttpCommon::SourceId source_id);
+    HttpInfractions* get_infractions(HttpCommon::SourceId);
 
     void set_one_hundred_response();
     bool final_response() const { return !second_response_expected; }
@@ -69,8 +69,9 @@ public:
     HttpTransaction* next = nullptr;
 
 private:
-    HttpTransaction(HttpFlowData* session_data_);
-    void discard_section(HttpMsgSection* section);
+    HttpTransaction(HttpFlowData*, snort::Flow* const);
+    void discard_section(HttpMsgSection*);
+    void publish_end_of_transaction();
 
     HttpFlowData* const session_data;
 
@@ -93,6 +94,9 @@ private:
     // parallel.
     bool shared_ownership = false;
 
+    unsigned pub_id;
+    snort::Flow* const flow;
+
     // Estimates of how much memory http_inspect uses to process a transaction
     static const uint16_t small_things = 400; // minor memory costs not otherwise accounted for
     static const uint16_t transaction_memory_usage_estimate;
index 6f9743e1fb44d40306f21c308881db81c5485f45..33889ccef67462a3aad5ba031678f376599fa711 100644 (file)
@@ -48,17 +48,6 @@ bool HttpVersionRuleOptModule::begin(const char*, int, SnortConfig*)
     return true;
 }
 
-static const std::map <std::string, VersionId> VersionStrToEnum =
-{
-    { "malformed", VERS__PROBLEMATIC },
-    { "other", VERS__OTHER },
-    { "1.0", VERS_1_0 },
-    { "1.1", VERS_1_1 },
-    { "2.0", VERS_2_0 },
-    { "3.0", VERS_3_0 },
-    { "0.9", VERS_0_9 }
-};
-
 bool HttpVersionRuleOptModule::parse_version_list(Value& v)
 {
     v.set_first_token();
index 305e795a87d18e66b1ff349e37b8d098423bd129..aba0218e77b1e0ae139568fe72c9ba203fc6ecb1 100644 (file)
 #include "config.h"
 #endif
 
+#include "pub_sub/http_transaction_end_event.h"
 #include "service_inspectors/http_inspect/http_common.h"
 #include "service_inspectors/http_inspect/http_enum.h"
 #include "service_inspectors/http_inspect/http_flow_data.h"
+#include "service_inspectors/http_inspect/http_inspect.h"
 #include "service_inspectors/http_inspect/http_module.h"
 #include "service_inspectors/http_inspect/http_transaction.h"
 #include "service_inspectors/http2_inspect/http2_flow_data.h"
 
+#include "http_unit_test_helpers.h"
+
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
 #include <CppUTestExt/MockSupport.h>
@@ -51,6 +55,22 @@ uint32_t str_to_hash(const uint8_t *, size_t) { return 0; }
 FlowData* Flow::get_flow_data(uint32_t) const { return nullptr; }
 int Flow::set_flow_data(FlowData*) { return 0;}
 Flow::~Flow() = default;
+unsigned DataBus::get_id(PubKey const&) { return 0; }
+void DataBus::publish(unsigned int, unsigned int, DataEvent&, Flow*) {}
+HttpTransactionEndEvent::HttpTransactionEndEvent(const HttpTransaction* const trans):
+    transaction(trans) {}
+Inspector::Inspector() { }
+Inspector::~Inspector() = default;
+bool Inspector::likes(Packet*) { return true; }
+bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return false; }
+class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
+const StreamBuffer StreamSplitter::reassemble(snort::Flow*, unsigned int, unsigned int, unsigned char const*, unsigned
+    int, unsigned int, unsigned int&)
+{
+    StreamBuffer buf { nullptr, 0 };
+    return buf;
+}
+unsigned StreamSplitter::max(snort::Flow*) { return 0; }
 }
 
 HttpParaList::UriParam::UriParam() {}
@@ -59,18 +79,41 @@ HttpParaList::~HttpParaList() {}
 
 unsigned Http2FlowData::inspector_id = 0;
 uint32_t Http2FlowData::get_processing_stream_id() const { return 0; }
+HttpInspect::HttpInspect(const HttpParaList* para) :
+    params(para), xtra_trueip_id(0), xtra_uri_id(0),
+    xtra_host_id(0), xtra_jsnorm_id(0)
+{ }
+HttpInspect::~HttpInspect() = default;
+bool HttpInspect::configure(SnortConfig*) { return true; }
+void HttpInspect::show(const SnortConfig*) const { }
+bool HttpInspect::get_buf(unsigned, snort::Packet*, snort::InspectionBuffer&) { return true; }
+HttpCommon::SectionType HttpInspect::get_type_expected(snort::Flow*, HttpCommon::SourceId) const
+{ return SEC_DISCARD; }
+void HttpInspect::finish_hx_body(snort::Flow*, HttpCommon::SourceId, HttpCommon::HXBodyState,
+    bool) const { }
+void HttpInspect::set_hx_body_state(snort::Flow*, HttpCommon::SourceId, HttpCommon::HXBodyState) const { }
+bool HttpInspect::get_fp_buf(snort::InspectionBuffer::Type, snort::Packet*,
+    snort::InspectionBuffer&) { return false; }
+void HttpInspect::eval(snort::Packet*) { }
+void HttpInspect::eval(snort::Packet*, HttpCommon::SourceId, const uint8_t*, uint16_t) { }
+void HttpInspect::clear(snort::Packet*) { }
+bool HttpInspect::get_buf(snort::InspectionBuffer::Type, snort::Packet*, snort::InspectionBuffer&) { return false; }
+const uint8_t* HttpInspect::adjust_log_packet(snort::Packet*, uint16_t&) { return nullptr; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Packet*, const uint8_t*, uint32_t, uint32_t, uint32_t*)
+{ return StreamSplitter::FLUSH; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Flow*, const uint8_t*, uint32_t, uint32_t*)
+{ return StreamSplitter::FLUSH; }
+const snort::StreamBuffer HttpStreamSplitter::reassemble(snort::Flow*, unsigned, unsigned, const
+    uint8_t*, unsigned, uint32_t, unsigned&)
+{
+    StreamBuffer buf { nullptr, 0 };
+    return buf;
+}
+bool HttpStreamSplitter::finish(snort::Flow*) { return false; }
+void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t) { }
 
 THREAD_LOCAL PegCount HttpModule::peg_counts[PEG_COUNT_MAX] = { };
 
-class HttpUnitTestSetup
-{
-public:
-    static SectionType* get_section_type(HttpFlowData* flow_data)
-        { assert(flow_data!=nullptr); return flow_data->section_type; }
-    static SectionType* get_type_expected(HttpFlowData* flow_data)
-        { assert(flow_data!=nullptr); return flow_data->type_expected; }
-};
-
 TEST_GROUP(http_transaction_test)
 {
     Flow* const flow = new Flow;
@@ -79,9 +122,14 @@ TEST_GROUP(http_transaction_test)
     SectionType* const section_type = HttpUnitTestSetup::get_section_type(flow_data);
     SectionType* const type_expected = HttpUnitTestSetup::get_type_expected(flow_data);
 
+    void setup() override
+    {
+        flow->gadget = new HttpInspect(&params);
+    }
     void teardown() override
     {
         delete flow_data;
+        delete flow->gadget;
         delete flow;
     }
 };
@@ -93,34 +141,34 @@ TEST(http_transaction_test, simple_transaction)
     // Request
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_BODY_CHUNK;
     section_type[SRC_CLIENT] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<100; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+      CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     }
     type_expected[SRC_CLIENT] = SEC_TRAILER;
     section_type[SRC_CLIENT] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     // Response
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<100; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, orphan_response)
@@ -128,17 +176,17 @@ TEST(http_transaction_test, orphan_response)
     // Response message without a request
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_SERVER] = SEC_STATUS;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow);
     CHECK(trans != nullptr);
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<10; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, simple_pipeline)
@@ -149,11 +197,11 @@ TEST(http_transaction_test, simple_pipeline)
     {
         type_expected[SRC_CLIENT] = SEC_REQUEST;
         section_type[SRC_CLIENT] = SEC_REQUEST;
-        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
         CHECK(trans[k] != nullptr);
         type_expected[SRC_CLIENT] = SEC_HEADER;
         section_type[SRC_CLIENT] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
         for (unsigned j=0; j < k; j++)
         {
             CHECK(trans[k] != trans[j]);
@@ -164,11 +212,11 @@ TEST(http_transaction_test, simple_pipeline)
     for (unsigned k=0; k < 4; k++)
     {
         section_type[SRC_SERVER] = SEC_STATUS;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_BODY_CL;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
 }
 
@@ -177,35 +225,35 @@ TEST(http_transaction_test, concurrent_request_response)
     // Response starts before request completes, request completes first
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_BODY_CHUNK;
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     section_type[SRC_CLIENT] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<4; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     }
     type_expected[SRC_CLIENT] = SEC_TRAILER;
     section_type[SRC_CLIENT] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<6; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, pipeline_underflow)
@@ -213,37 +261,37 @@ TEST(http_transaction_test, pipeline_underflow)
     // Underflow scenario with request, two responses, request, response
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER);
+    trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow);
     CHECK(trans != nullptr);
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans2 = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans2 = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK((trans2 != nullptr) && (trans2 != trans));
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER);
+    trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow);
     CHECK((trans != nullptr) && (trans != trans2));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, concurrent_request_response_underflow)
@@ -251,47 +299,47 @@ TEST(http_transaction_test, concurrent_request_response_underflow)
     // Response starts before request completes, response completes first, second response
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_BODY_CHUNK;
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<6; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     section_type[SRC_CLIENT] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<4; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     }
     type_expected[SRC_CLIENT] = SEC_TRAILER;
     section_type[SRC_CLIENT] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     section_type[SRC_SERVER] = SEC_STATUS;
-    HttpTransaction* trans2 = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER);
+    HttpTransaction* trans2 = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow);
     CHECK((trans2 != nullptr) && (trans2 != trans));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<6; k++)
     {
-        CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans2 == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, basic_continue)
@@ -300,43 +348,43 @@ TEST(http_transaction_test, basic_continue)
     // Request headers
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_BODY_CHUNK;
 
     // Interim response
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     trans->set_one_hundred_response();
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     // Request body
     section_type[SRC_CLIENT] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<4; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     }
     type_expected[SRC_CLIENT] = SEC_TRAILER;
     section_type[SRC_CLIENT] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     // Second response
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<6; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, multiple_continue)
@@ -345,46 +393,46 @@ TEST(http_transaction_test, multiple_continue)
     // Request headers
     type_expected[SRC_CLIENT] = SEC_REQUEST;
     section_type[SRC_CLIENT] = SEC_REQUEST;
-    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+    HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
     CHECK(trans != nullptr);
     type_expected[SRC_CLIENT] = SEC_HEADER;
     section_type[SRC_CLIENT] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_BODY_CHUNK;
 
     // Interim responses
     for (unsigned k=0; k < 10; k++)
     {
         section_type[SRC_SERVER] = SEC_STATUS;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         trans->set_one_hundred_response();
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
 
     // Request body
     section_type[SRC_CLIENT] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<4; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     }
     type_expected[SRC_CLIENT] = SEC_TRAILER;
     section_type[SRC_CLIENT] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
     type_expected[SRC_CLIENT] = SEC_REQUEST;
 
     // Final response
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     section_type[SRC_SERVER] = SEC_BODY_CHUNK;
     for (unsigned k=0; k<6; k++)
     {
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
     section_type[SRC_SERVER] = SEC_TRAILER;
-    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 }
 
 TEST(http_transaction_test, multiple_orphan_continue)
@@ -395,25 +443,25 @@ TEST(http_transaction_test, multiple_orphan_continue)
     {
         // Interim response
         section_type[SRC_SERVER] = SEC_STATUS;
-        HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER);
+        HttpTransaction* trans = HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow);
         CHECK(trans != nullptr);
         trans->set_one_hundred_response();
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_BODY_CHUNK;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_TRAILER;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
         // Final response
         section_type[SRC_SERVER] = SEC_STATUS;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_BODY_CHUNK;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_TRAILER;
-        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
 }
 
@@ -427,11 +475,11 @@ TEST(http_transaction_test, pipeline_continue_pipeline)
     {
         type_expected[SRC_CLIENT] = SEC_REQUEST;
         section_type[SRC_CLIENT] = SEC_REQUEST;
-        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
         CHECK(trans[k] != nullptr);
         type_expected[SRC_CLIENT] = SEC_HEADER;
         section_type[SRC_CLIENT] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
         for (unsigned j=0; j < k; j++)
         {
             CHECK(trans[k] != trans[j]);
@@ -443,34 +491,34 @@ TEST(http_transaction_test, pipeline_continue_pipeline)
     for (unsigned k=0; k < 3; k++)
     {
         section_type[SRC_SERVER] = SEC_STATUS;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_BODY_CL;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
 
     // Interim response to fourth request
     section_type[SRC_SERVER] = SEC_STATUS;
-    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     trans[3]->set_one_hundred_response();
     section_type[SRC_SERVER] = SEC_HEADER;
-    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
 
     // Finish the fourth request
     section_type[SRC_CLIENT] = SEC_BODY_CL;
-    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+    CHECK(trans[3] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
 
     // Requests 5-7 in pipeline
     for (unsigned k=4; k < 7; k++)
     {
         type_expected[SRC_CLIENT] = SEC_REQUEST;
         section_type[SRC_CLIENT] = SEC_REQUEST;
-        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT);
+        trans[k] = HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow);
         CHECK(trans[k] != nullptr);
         type_expected[SRC_CLIENT] = SEC_HEADER;
         section_type[SRC_CLIENT] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_CLIENT, flow));
         for (unsigned j=5; j < k; j++)
         {
             CHECK(trans[k] != trans[j]);
@@ -482,11 +530,11 @@ TEST(http_transaction_test, pipeline_continue_pipeline)
     for (unsigned k=3; k < 7; k++)
     {
         section_type[SRC_SERVER] = SEC_STATUS;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_HEADER;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
         section_type[SRC_SERVER] = SEC_BODY_CL;
-        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER));
+        CHECK(trans[k] == HttpTransaction::attach_my_transaction(flow_data, SRC_SERVER, flow));
     }
 }
 
diff --git a/src/service_inspectors/http_inspect/test/http_unit_test_helpers.h b/src/service_inspectors/http_inspect/test/http_unit_test_helpers.h
new file mode 100644 (file)
index 0000000..3b9c16f
--- /dev/null
@@ -0,0 +1,36 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_unit_test_helpers.h author Maya Dagon <mdagon@cisco.com>
+// Code moved from http_transaction_test.cc, author Tom Peters <thopeter@cisco.com>
+
+#ifndef HTTP_UNIT_TEST_HELPERS_H
+#define HTTP_UNIT_TEST_HELPERS_H
+
+#include "service_inspectors/http_inspect/http_common.h"
+#include "service_inspectors/http_inspect/http_flow_data.h"
+
+class HttpUnitTestSetup
+{
+public:
+    static HttpCommon::SectionType* get_section_type(HttpFlowData* flow_data)
+        { assert(flow_data!=nullptr); return flow_data->section_type; }
+    static HttpCommon::SectionType* get_type_expected(HttpFlowData* flow_data)
+        { assert(flow_data!=nullptr); return flow_data->type_expected; }
+};
+
+#endif