From: Maya Dagon (mdagon) Date: Tue, 28 Jan 2025 17:49:41 +0000 (+0000) Subject: Pull request #4557: extractor: conn.log support X-Git-Tag: 3.6.3.0~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3716542d98607eddc18ea2b36725ae8104056ae5;p=thirdparty%2Fsnort3.git Pull request #4557: extractor: conn.log support Merge in SNORT/snort3 from ~MDAGON/snort3:conn_sub to master Squashed commit of the following: commit d6d6945d5c52d77ff401201b6e6112348002dc57 Author: maya dagon Date: Fri Nov 8 13:50:25 2024 -0500 extractor: support connection logs --- diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt index 872324ca5..a05384945 100644 --- a/doc/user/extractor.txt +++ b/doc/user/extractor.txt @@ -31,7 +31,8 @@ things it allows tenants to get independent data logging configurations. { { service = 'http', tenant_id = 1, on_events = 'eot', fields = 'ts, uri, host, method' }, { service = 'ftp', tenant_id = 1, on_events = 'request', fields = 'ts, command, arg' }, - { service = 'http', tenant_id = 2, on_events = 'eot', fields = 'ts, uri' } + { service = 'http', tenant_id = 2, on_events = 'eot', fields = 'ts, uri' }, + { service = 'conn', tenant_id = 1, on_events = 'eof', fields = 'ts, uid, service' } } } @@ -45,6 +46,8 @@ Services and their events: ** request ** response ** eot (a session defined by the following commands: APPE, DELE, RETR, STOR, STOU, ACCT, PORT, PASV, EPRT, EPSV) +* connection (conn) + ** eof (end of flow) Common fields available for every service: @@ -82,6 +85,14 @@ Fields supported for FTP: * `data_channel.resp_h` - IP address of data channel receiving point * `data_channel.resp_p` - TCP port of data channel receiving point +Fields supported for connection: + +* `duration` - connection duration in seconds +* `proto` - transport layer protocol of the connection +* `service` - connection's application protocol +* `orig_pkts` - number of packets originator sent +* `resp_pkts` - number of packets responder sent + ==== Example Adding the following lines to a default snort configuration (which supports FTP diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index ffe4691a8..cd6dbe0aa 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -1,6 +1,7 @@ set( FILE_LIST extractor.cc extractor.h + extractor_conn.cc extractor_csv_logger.cc extractor_csv_logger.h extractor_enums.h diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index ac752ec95..f7050b7b3 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -50,7 +50,7 @@ THREAD_LOCAL ExtractorLogger* Extractor::logger = nullptr; static const Parameter extractor_proto_params[] = { - { "service", Parameter::PT_ENUM, "http | ftp", nullptr, + { "service", Parameter::PT_ENUM, "http | ftp | conn", nullptr, "service to extract from" }, { "tenant_id", Parameter::PT_INT, "0:max32", "0", diff --git a/src/network_inspectors/extractor/extractor_conn.cc b/src/network_inspectors/extractor/extractor_conn.cc new file mode 100644 index 000000000..64c6ec7fb --- /dev/null +++ b/src/network_inspectors/extractor/extractor_conn.cc @@ -0,0 +1,170 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// extractor_conn.cc author Maya Dagon + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "extractor_conn.h" + +#include "detection/detection_engine.h" +#include "flow/flow_key.h" +#include "profiler/profiler.h" +#include "pub_sub/intrinsic_event_ids.h" +#include "sfip/sf_ip.h" +#include "utils/util.h" +#include "utils/util_net.h" + +#include "extractor.h" +#include "extractor_enums.h" + +using namespace snort; +using namespace std; + +static uint64_t get_orig_pkts(const DataEvent*, const Packet*, const Flow* f) +{ + return f->flowstats.client_pkts; +} + +static uint64_t get_resp_pkts(const DataEvent*, const Packet*, const Flow* f) +{ + return f->flowstats.server_pkts; +} + +static uint64_t get_duration(const DataEvent*, const Packet*, const Flow* f) +{ + return f->last_data_seen - f->flowstats.start_time.tv_sec; +} + +static const map sub_num_getters = +{ + {"orig_pkts", get_orig_pkts}, + {"resp_pkts", get_resp_pkts}, + {"duration", get_duration} +}; + +static const char* get_service(const DataEvent*, const Packet*, const Flow* f) +{ + SnortConfig* sc = SnortConfig::get_main_conf(); + return sc->proto_ref->get_name(f->ssn_state.snort_protocol_id); +} + +static const map pkttype_to_protocol = +{ + {PktType::TCP, "TCP"}, + {PktType::UDP, "UDP"}, + {PktType::IP, "IP"}, + {PktType::ICMP, "ICMP"} +}; + +static const char* get_proto(const DataEvent*, const Packet*, const Flow* f) +{ + const auto& iter = pkttype_to_protocol.find(f->pkt_type); + return (iter != pkttype_to_protocol.end()) ? iter->second.c_str() : ""; +} + +static const map sub_buf_getters = +{ + {"proto", get_proto}, + {"service", get_service} +}; + +THREAD_LOCAL const snort::Connector::ID* ConnExtractor::log_id = nullptr; + +ConnExtractor::ConnExtractor(Extractor& i, uint32_t t, const vector& fields) + : ExtractorEvent(i, t) +{ + for (const auto& f : fields) + { + if (append(nts_fields, nts_getters, f)) + continue; + if (append(sip_fields, sip_getters, f)) + continue; + if (append(num_fields, num_getters, f)) + continue; + if (append(num_fields, sub_num_getters, f)) + continue; + if (append(buf_fields, sub_buf_getters, f)) + continue; + } + + DataBus::subscribe(intrinsic_pub_key, IntrinsicEventIds::FLOW_END, new Eof(*this, S_NAME)); +} + +void ConnExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + +void ConnExtractor::handle(DataEvent& event, Flow* flow) +{ + // cppcheck-suppress unreadVariable + Profile profile(extractor_perf_stats); + + uint32_t tid = 0; + + if ((flow->pkt_type < PktType::IP) or (flow->pkt_type > PktType::ICMP)) + return; + +#ifndef DISABLE_TENANT_ID + tid = flow->key->tenant_id; +#endif + + if (tenant_id != tid) + return; + + Packet* packet = (DetectionEngine::get_context()) ? DetectionEngine::get_current_packet() : nullptr; + + extractor_stats.total_event++; + + logger->open_record(); + log(nts_fields, &event, packet, flow); + log(sip_fields, &event, packet, flow); + log(num_fields, &event, packet, flow); + log(buf_fields, &event, packet, flow); + logger->close_record(*log_id); +} + +//------------------------------------------------------------------------- +// Unit Tests +//------------------------------------------------------------------------- + +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +#include + +TEST_CASE("Conn Proto", "[extractor]") +{ + Flow* flow = new Flow; + InspectionPolicy ins; + set_inspection_policy(&ins); + NetworkPolicy net; + set_network_policy(&net); + + SECTION("unknown") + { + flow->pkt_type = PktType::NONE; + const char* proto = get_proto(nullptr, nullptr, flow); + CHECK_FALSE(strcmp("", proto)); + } + + delete flow; +} + +#endif diff --git a/src/network_inspectors/extractor/extractor_conn.h b/src/network_inspectors/extractor/extractor_conn.h new file mode 100644 index 000000000..83462a757 --- /dev/null +++ b/src/network_inspectors/extractor/extractor_conn.h @@ -0,0 +1,43 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// extractor_conn.h author Maya Dagon + +#ifndef EXTRACTOR_CONN_H +#define EXTRACTOR_CONN_H + +#include "extractors.h" + +class ConnExtractor : public ExtractorEvent +{ +public: + using ConnNumGetFn = uint64_t (*) (const DataEvent*, const Packet*, const Flow*); + using ConnNumField = DataField; + + ConnExtractor(Extractor&, uint32_t tenant, const std::vector& fields); + + void handle(DataEvent&, Flow*); + +private: + using Eof = Handler; + + void internal_tinit(const snort::Connector::ID*) override; + + static THREAD_LOCAL const snort::Connector::ID* log_id; +}; + +#endif diff --git a/src/network_inspectors/extractor/extractor_enums.h b/src/network_inspectors/extractor/extractor_enums.h index ea19e43e5..c53838713 100644 --- a/src/network_inspectors/extractor/extractor_enums.h +++ b/src/network_inspectors/extractor/extractor_enums.h @@ -29,6 +29,7 @@ public: { HTTP, FTP, + CONN, UNDEFINED, MAX }; @@ -48,6 +49,8 @@ public: return "http"; case FTP: return "ftp"; + case CONN: + return "conn"; case UNDEFINED: // fallthrough case MAX: // fallthrough default: diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index 92945a734..85746507a 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -26,6 +26,7 @@ #include "log/messages.h" #include "extractor.h" +#include "extractor_conn.h" #include "extractor_ftp.h" #include "extractor_http.h" @@ -122,6 +123,10 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo srv = new FtpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); break; + case ServiceType::CONN: + srv = new ConnExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); + break; + case ServiceType::UNDEFINED: // fallthrough default: ParseError("'%s' service is not supported", cfg.service.c_str()); @@ -206,9 +211,6 @@ HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector& srv_fields, + const std::vector& srv_events, ServiceType s_type, Extractor& ins) + : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, ins) +{ + for (const auto& event : get_events()) + { + if (!strcmp("eof", event.c_str())) + handlers.push_back(new ConnExtractor(ins, tenant_id, get_fields())); + } +} + +const snort::Connector::ID& ConnExtractorService::internal_tinit() +{ return log_id = logger->get_id(type.c_str()); } + +const snort::Connector::ID& ConnExtractorService::get_log_id() +{ return log_id; } + //------------------------------------------------------------------------- // Unit Tests //------------------------------------------------------------------------- @@ -286,11 +325,13 @@ TEST_CASE("Service Type", "[extractor]") { ServiceType http = ServiceType::HTTP; ServiceType ftp = ServiceType::FTP; + ServiceType conn = ServiceType::CONN; ServiceType undef = ServiceType::UNDEFINED; ServiceType max = ServiceType::MAX; CHECK_FALSE(strcmp("http", http.c_str())); CHECK_FALSE(strcmp("ftp", ftp.c_str())); + CHECK_FALSE(strcmp("conn", conn.c_str())); CHECK_FALSE(strcmp("(not set)", undef.c_str())); CHECK_FALSE(strcmp("(not set)", max.c_str())); } diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h index 95857b32b..91a0625b9 100644 --- a/src/network_inspectors/extractor/extractor_service.h +++ b/src/network_inspectors/extractor/extractor_service.h @@ -113,5 +113,19 @@ private: static THREAD_LOCAL snort::Connector::ID log_id; }; +class ConnExtractorService : public ExtractorService +{ +public: + ConnExtractorService(uint32_t tenant, const std::vector& fields, + const std::vector& events, ServiceType, Extractor&); + +private: + const snort::Connector::ID& internal_tinit() override; + const snort::Connector::ID& get_log_id() override; + + static ServiceBlueprint blueprint; + static THREAD_LOCAL snort::Connector::ID log_id; +}; + #endif diff --git a/src/network_inspectors/extractor/extractors.h b/src/network_inspectors/extractor/extractors.h index 402d0e3a1..3e344cd90 100644 --- a/src/network_inspectors/extractor/extractors.h +++ b/src/network_inspectors/extractor/extractors.h @@ -28,6 +28,7 @@ #include "framework/data_bus.h" #include "framework/connector.h" #include "sfip/sf_ip.h" +#include "time/packet_time.h" #include "extractor_logger.h" @@ -88,7 +89,14 @@ protected: }; static struct timeval get_timestamp(const DataEvent*, const Packet* p, const Flow*) - { return p->pkth->ts; } + { + if (p != nullptr) + return p->pkth->ts; + + struct timeval timestamp; + snort::packet_gettimeofday(×tamp); + return timestamp; + } static const SfIp& get_ip_src(const DataEvent*, const Packet*, const Flow* flow) { return flow->flags.client_initiated ? flow->client_ip : flow->server_ip; } @@ -103,7 +111,7 @@ protected: { return flow->server_port; } static uint64_t get_pkt_num(const DataEvent*, const Packet* p, const Flow*) - { return p->context->packet_number; } + { return (p != nullptr) ? p->context->packet_number : 0; } static uint64_t get_uid(const DataEvent*, const Packet*, const Flow* flow) { return ExtractorEvent::get_hash().do_hash((const unsigned char*)flow->key, 0); }