From: Russ Combs (rucombs) Date: Thu, 10 Dec 2015 00:58:27 +0000 (-0500) Subject: Merge pull request #183 in SNORT/snort3 from iprep_final to master X-Git-Tag: 3.0.0-233~685 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=0459f69780b2b1d6180b01cdfd9feb3c4d620f32;p=thirdparty%2Fsnort3.git Merge pull request #183 in SNORT/snort3 from iprep_final to master Squashed commit of the following: commit 208b107cc762657c36fa703f9129f7b8d607319a Author: Bhagyashree Bantwal Date: Mon Dec 7 17:46:04 2015 -0500 IP Reputation Rewrite without shared memory fixes to iprep reputation changes per review Update flow state based on iprep decision move profile stats to start of eval in reputation reputation changes/fixes move profile stats to start of eval in reputation --- diff --git a/configure.ac b/configure.ac index ae0c1347f..ea7a81c27 100644 --- a/configure.ac +++ b/configure.ac @@ -992,6 +992,7 @@ src/network_inspectors/binder/Makefile \ src/network_inspectors/normalize/Makefile \ src/network_inspectors/perf_monitor/Makefile \ src/network_inspectors/port_scan/Makefile \ +src/network_inspectors/reputation/Makefile \ src/packet_io/Makefile \ src/parser/Makefile \ src/piglet/Makefile \ diff --git a/lua/snort.lua b/lua/snort.lua index d60331ed2..c7286e224 100644 --- a/lua/snort.lua +++ b/lua/snort.lua @@ -74,6 +74,7 @@ smtp = { } perf_monitor = { } pop = { } port_scan = { } +reputation = { } rpc_decode = { } sip = { } ssh = { } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6e3bc1c0e..6a38a59be 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -99,6 +99,7 @@ target_link_libraries( snort service_inspectors ${STATIC_INSPECTOR_LIBRARIES} port_scan + reputation stream stream_base stream_ip diff --git a/src/Makefile.am b/src/Makefile.am index a98cfe652..1a8fcc79c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -58,6 +58,7 @@ network_inspectors/libnetwork_inspectors.a \ network_inspectors/binder/libbinder.a \ network_inspectors/normalize/libnormalize.a \ network_inspectors/perf_monitor/libperf_monitor.a \ +network_inspectors/reputation/libreputation.a \ service_inspectors/libservice_inspectors.a \ $(lib_list) \ service_inspectors/imap/libimap.a \ diff --git a/src/flow/flow.h b/src/flow/flow.h index 022ba3aff..06aff8905 100644 --- a/src/flow/flow.h +++ b/src/flow/flow.h @@ -270,6 +270,16 @@ public: data = nullptr; } + void disable_inspection() + { + disable_inspect = true; + } + + bool is_inspection_disabled() + { + return disable_inspect; + } + public: // FIXIT-M privatize if possible // these fields are const after initialization const FlowKey* key; @@ -314,6 +324,7 @@ public: // FIXIT-M privatize if possible uint8_t outer_client_ttl, outer_server_ttl; uint8_t response_count; + bool disable_inspect; public: LwState ssn_state; diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc index e3ec1adba..bda47aa37 100644 --- a/src/flow/flow_control.cc +++ b/src/flow/flow_control.cc @@ -416,6 +416,7 @@ unsigned FlowControl::process(Flow* flow, Packet* p) unsigned news = 0; p->flow = flow; + p->disable_inspect = flow->is_inspection_disabled(); if ( flow->flow_state ) set_policies(snort_conf, flow->policy_id); diff --git a/src/log/obfuscation.cc b/src/log/obfuscation.cc index 7b0cf2d33..c0e6c4dca 100644 --- a/src/log/obfuscation.cc +++ b/src/log/obfuscation.cc @@ -1362,7 +1362,8 @@ int main(int argc, char* argv[]) packet.pseudo_type = 0; packet.user_policy_id = 0; packet.iplist_id = 0; - packeet.ps_proto = 0; + packet.ps_proto = 0; + packet.disable_inspect = false; pkthtmp = (DAQ_PktHdr_t*)&packet.pkth; pkthtmp = &pkth; diff --git a/src/main/snort_debug.h b/src/main/snort_debug.h index 195d94f02..d32b19d92 100644 --- a/src/main/snort_debug.h +++ b/src/main/snort_debug.h @@ -70,6 +70,7 @@ #define DEBUG_SIP 0x0000100000000000LL #define DEBUG_SSL 0x0000200000000000LL #define DEBUG_SMTP 0x0000400000000000LL +#define DEBUG_REPUTATION 0x0000800000000000LL #define DEBUG_CODEC 0x0001000000000000LL #define DEBUG_INSPECTOR 0x0002000000000000LL diff --git a/src/managers/inspector_manager.cc b/src/managers/inspector_manager.cc index 8d72607ce..29e40f156 100644 --- a/src/managers/inspector_manager.cc +++ b/src/managers/inspector_manager.cc @@ -759,7 +759,7 @@ void InspectorManager::bumble(Packet* p) flow->session->restart(p); } -void InspectorManager::full_inspection(FrameworkPolicy* fp, Packet* p) +bool InspectorManager::full_inspection(FrameworkPolicy* fp, Packet* p) { Flow* flow = p->flow; @@ -769,7 +769,10 @@ void InspectorManager::full_inspection(FrameworkPolicy* fp, Packet* p) else if ( flow->clouseau and !p->is_cooked() ) bumble(p); - if ( !p->dsize ) + if( p->disable_inspect ) + return false; + + else if ( !p->dsize ) DisableDetect(p); else if ( flow->gadget && flow->gadget->likes(p) ) @@ -777,6 +780,8 @@ void InspectorManager::full_inspection(FrameworkPolicy* fp, Packet* p) flow->gadget->eval(p); s_clear = true; } + + return true; } void InspectorManager::execute(Packet* p) @@ -791,13 +796,19 @@ void InspectorManager::execute(Packet* p) if ( !p->has_paf_payload() ) ::execute(p, fp->session.vec, fp->session.num); + if( p->disable_inspect ) + return; + Flow* flow = p->flow; if ( !flow ) ::execute(p, fp->network.vec, fp->network.num); else if ( flow->full_inspection() ) - full_inspection(fp, p); + { + if(!full_inspection(fp, p)) + return; + } ::execute(p, fp->probe.vec, fp->probe.num); } diff --git a/src/managers/inspector_manager.h b/src/managers/inspector_manager.h index b78218c0b..a4920fa6f 100644 --- a/src/managers/inspector_manager.h +++ b/src/managers/inspector_manager.h @@ -92,7 +92,7 @@ public: private: static void bumble(Packet*); - static void full_inspection(FrameworkPolicy*, Packet*); + static bool full_inspection(FrameworkPolicy*, Packet*); }; #endif diff --git a/src/network_inspectors/CMakeLists.txt b/src/network_inspectors/CMakeLists.txt index 7fe796299..5271c4480 100644 --- a/src/network_inspectors/CMakeLists.txt +++ b/src/network_inspectors/CMakeLists.txt @@ -4,6 +4,7 @@ add_subdirectory(binder) add_subdirectory(normalize) add_subdirectory(perf_monitor) add_subdirectory(port_scan) +add_subdirectory(reputation) if(STATIC_INSPECTORS) set(STATIC_INSPECTOR_LIBS @@ -22,5 +23,6 @@ target_link_libraries( network_inspectors binder perf_monitor normalize + reputation stream_tcp ) diff --git a/src/network_inspectors/Makefile.am b/src/network_inspectors/Makefile.am index 38e093cf9..492a42c96 100644 --- a/src/network_inspectors/Makefile.am +++ b/src/network_inspectors/Makefile.am @@ -18,5 +18,5 @@ arp_spoof \ binder \ normalize \ perf_monitor \ -port_scan - +port_scan \ +reputation diff --git a/src/network_inspectors/network_inspectors.cc b/src/network_inspectors/network_inspectors.cc index 7d8a1366f..20ab7789e 100644 --- a/src/network_inspectors/network_inspectors.cc +++ b/src/network_inspectors/network_inspectors.cc @@ -28,6 +28,7 @@ extern const BaseApi* nin_normalize; extern const BaseApi* nin_perf_monitor; extern const BaseApi* nin_port_scan_global; extern const BaseApi* nin_port_scan; +extern const BaseApi* nin_reputation; #ifdef STATIC_INSPECTORS extern const BaseApi* nin_arp_spoof; @@ -40,6 +41,7 @@ const BaseApi* network_inspectors[] = nin_perf_monitor, nin_port_scan_global, nin_port_scan, + nin_reputation, #ifdef STATIC_INSPECTORS nin_arp_spoof, diff --git a/src/network_inspectors/reputation/CMakeLists.txt b/src/network_inspectors/reputation/CMakeLists.txt new file mode 100644 index 000000000..c465460cb --- /dev/null +++ b/src/network_inspectors/reputation/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_library( reputation STATIC + reputation_config.h + reputation_inspect.h + reputation_inspect.cc + reputation_module.cc + reputation_module.h + reputation_parse.cc + reputation_parse.h +) + diff --git a/src/network_inspectors/reputation/Makefile.am b/src/network_inspectors/reputation/Makefile.am new file mode 100644 index 000000000..c0e1d7d52 --- /dev/null +++ b/src/network_inspectors/reputation/Makefile.am @@ -0,0 +1,11 @@ + +noinst_LIBRARIES = libreputation.a + +libreputation_a_SOURCES = \ +reputation_config.h \ +reputation_inspect.h \ +reputation_inspect.cc \ +reputation_module.cc \ +reputation_module.h \ +reputation_parse.h \ +reputation_parse.cc diff --git a/src/network_inspectors/reputation/dev_notes.txt b/src/network_inspectors/reputation/dev_notes.txt new file mode 100644 index 000000000..5569be6d3 --- /dev/null +++ b/src/network_inspectors/reputation/dev_notes.txt @@ -0,0 +1,6 @@ +This directory contains all files related to IP Reputation inspection. + +Reputation inspector provides basic IP blacklist/whitelist capabilities, to +block/drop/pass traffic from IP addresses listed. In the past, we use standard +Snort rules to implement Reputation-based IP blocking. This inspector will +address the performance issue and make the IP reputation management easier. diff --git a/src/network_inspectors/reputation/reputation_config.h b/src/network_inspectors/reputation/reputation_config.h new file mode 100644 index 000000000..261873298 --- /dev/null +++ b/src/network_inspectors/reputation/reputation_config.h @@ -0,0 +1,103 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. +// Copyright (C) 2004-2013 Sourcefire, Inc. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +#ifndef REPUTATION_CONFIG_H +#define REPUTATION_CONFIG_H + +#include "main/snort_types.h" +#include "sfrt/sfrt_flat.h" +#include "main/snort_debug.h" +#include "framework/counts.h" +#include "main/thread.h" + +#define NUM_INDEX_PER_ENTRY 4 + +// Configuration for reputation network inspector + +enum NestedIP +{ + INNER, + OUTER, + ALL +}; + +enum WhiteAction +{ + UNBLACK, + TRUST +}; + +enum IPdecision +{ + DECISION_NULL, + BLACKLISTED, + WHITELISTED_TRUST, + MONITORED, + WHITELISTED_UNBLACK, + DECISION_MAX +}; + +struct ListInfo +{ + uint8_t listIndex; + uint8_t listType; + uint32_t listId; +}; + +struct ReputationConfig +{ + uint32_t memcap = 500; + int numEntries = 0; + bool scanlocal = false; + IPdecision priority = WHITELISTED_TRUST; + NestedIP nestedIP = INNER; + WhiteAction whiteAction = UNBLACK; + MEM_OFFSET local_black_ptr = 0; + MEM_OFFSET local_white_ptr = 0; + uint8_t* reputation_segment = nullptr; + char* blacklist_path = nullptr; + char* whitelist_path = nullptr; + bool memCapReached = false; + table_flat_t* iplist = nullptr; + ListInfo* listInfo = nullptr; + + ~ReputationConfig(); +}; + +struct IPrepInfo +{ + char listIndexes[NUM_INDEX_PER_ENTRY]; + MEM_OFFSET next; +}; + +DEBUG_WRAP(void ReputationPrintRepInfo(IPrepInfo* repInfo, uint8_t* base); ) + +struct ReputationStats +{ + PegCount packets; + PegCount blacklisted; + PegCount whitelisted; + PegCount monitored; + PegCount memory_allocated; +}; + +extern const PegInfo reputation_peg_names[]; +extern ReputationStats reputationstats; +#endif + diff --git a/src/network_inspectors/reputation/reputation_inspect.cc b/src/network_inspectors/reputation/reputation_inspect.cc new file mode 100644 index 000000000..ef3ecc6fd --- /dev/null +++ b/src/network_inspectors/reputation/reputation_inspect.cc @@ -0,0 +1,460 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. +// Copyright (C) 2004-2013 Sourcefire, Inc. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// reputation_inspect.cc author Hui Cao + +#include "reputation_inspect.h" + +#include "reputation_module.h" +#include "reputation_parse.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "protocols/packet.h" +#include "sfip/sf_ip.h" +#include "events/event_queue.h" +#include "main/snort_types.h" +#include "main/snort_debug.h" +#include "profiler/profiler.h" +#include "stream/stream_api.h" +#include "file_api/file_api.h" +#include "parser/parser.h" +#include "framework/inspector.h" +#include "utils/sfsnprintfappend.h" +#include "target_based/snort_protocols.h" +#include "detection/detect.h" +#include "packet_io/active.h" + +THREAD_LOCAL ProfileStats reputationPerfStats; +ReputationStats reputationstats; + +const PegInfo reputation_peg_names[] = +{ + { "packets", "total packets processed" }, + { "blacklisted", "number of packets blacklisted" }, + { "whitelisted", "number of packets whitelisted" }, + { "monitored", "number of packets monitored" }, + { "memory_allocated", "total memory allocated" }, + + { nullptr, nullptr } +}; + +const char* NestedIPKeyword[] = +{ + "inner", + "outer", + "all", + nullptr +}; + +const char* WhiteActionOption[] = +{ + "unblack", + "trust", + nullptr +}; + +/* + * Function prototype(s) + */ +static void snort_reputation(ReputationConfig* GlobalConf, Packet* p); + +unsigned ReputationFlowData::flow_id = 0; + +ReputationData* SetNewReputationData(Flow* flow) +{ + ReputationFlowData* fd = new ReputationFlowData; + flow->set_application_data(fd); + return &fd->session; +} + +static ReputationData* get_session_data(Flow* flow) +{ + ReputationFlowData* fd = (ReputationFlowData*)flow->get_application_data( + ReputationFlowData::flow_id); + + return fd ? &fd->session : nullptr; +} + +static bool IsReputationDisabled(Flow* flow) +{ + ReputationData* data; + + if (!flow) + return false; + + data = get_session_data(flow); + + if (!data) + SetNewReputationData(flow); + + return data ? data->disabled : false; +} + +static void DisableReputation(Flow* flow) +{ + ReputationData* data; + + if (!flow) + return; + + data = get_session_data(flow); + + if (data) + data->disabled = true; +} + +void PrintIPlistStats(ReputationConfig* config) +{ + /*Print out the summary*/ + LogMessage(" Reputation total memory usage: " STDu64 " bytes\n", + reputationstats.memory_allocated); + config->numEntries = sfrt_flat_num_entries(config->iplist); + LogMessage(" Reputation total entries loaded: %u, invalid: %lu, re-defined: %lu\n", + config->numEntries,total_invalids,total_duplicates); +} + +void PrintReputationConf(ReputationConfig* config) +{ + assert(config); + + PrintIPlistStats(config); + + LogMessage(" Memcap: %d %s \n", + config->memcap, + config->memcap == 500 ? "(Default) M bytes" : "M bytes"); + LogMessage(" Scan local network: %s\n", + config->scanlocal ? "ENABLED" : "DISABLED (Default)"); + LogMessage(" Reputation priority: %s \n", + config->priority == WHITELISTED_TRUST ? + "whitelist (Default)" : "blacklist"); + LogMessage(" Nested IP: %s %s \n", + NestedIPKeyword[config->nestedIP], + config->nestedIP == INNER ? "(Default)" : ""); + LogMessage(" White action: %s %s \n", + WhiteActionOption[config->whiteAction], + config->whiteAction == UNBLACK ? "(Default)" : ""); + if (config->blacklist_path) + LogMessage(" Blacklist File Path: %s\n", config->blacklist_path); + + if (config->whitelist_path) + LogMessage(" Whitelist File Path: %s\n", config->whitelist_path); + + LogMessage("\n"); +} + +static inline IPrepInfo* ReputationLookup(ReputationConfig* config, const sfip_t* ip) +{ + IPrepInfo* result; + + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Lookup address: %s \n",sfip_to_str(ip) ); ); + if (!config->scanlocal) + { + if (sfip_is_private(ip) ) + { + DEBUG_WRAP(DebugMessage(DEBUG_REPUTATION, "Private address\n"); ); + return nullptr; + } + } + + result = (IPrepInfo*)sfrt_flat_dir8x_lookup((void*)ip, config->iplist); + + return (result); +} + +static inline IPdecision GetReputation(ReputationConfig* config, IPrepInfo* repInfo, + uint32_t* listid) +{ + IPdecision decision = DECISION_NULL; + uint8_t* base; + ListInfo* listInfo; + + /*Walk through the IPrepInfo lists*/ + base = (uint8_t*)config->iplist; + listInfo = (ListInfo*)(&base[config->iplist->list_info]); + + while (repInfo) + { + int i; + for (i = 0; i < NUM_INDEX_PER_ENTRY; i++) + { + int list_index = repInfo->listIndexes[i]; + if (!list_index) + break; + list_index--; + if (WHITELISTED_UNBLACK == (IPdecision)listInfo[list_index].listType) + return DECISION_NULL; + if (config->priority == (IPdecision)listInfo[list_index].listType ) + { + *listid = listInfo[list_index].listId; + return ((IPdecision)listInfo[list_index].listType); + } + else if ( decision < listInfo[list_index].listType) + { + decision = (IPdecision)listInfo[list_index].listType; + *listid = listInfo[list_index].listId; + } + } + + if (!repInfo->next) + break; + repInfo = (IPrepInfo*)(&base[repInfo->next]); + } + + return decision; +} + +static bool ReputationDecisionPerLayer(ReputationConfig* config, Packet* p, ip::IpApi ip_api, IPdecision* decision_final) +{ + const sfip_t* ip; + IPdecision decision; + IPrepInfo* result; + + ip = ip_api.get_src(); + result = ReputationLookup(config, ip); + if (result) + { + decision = GetReputation(config, result, &p->iplist_id); + + *decision_final = decision; + if ( config->priority == decision) + return true; + } + + ip = ip_api.get_dst(); + result = ReputationLookup(config, ip); + if (result) + { + decision = GetReputation(config, result, &p->iplist_id); + + *decision_final = decision; + if ( config->priority == decision) + return true; + } + + return false; +} + +static IPdecision ReputationDecision(ReputationConfig* config, Packet* p) +{ + IPdecision decision_final = DECISION_NULL; + + ip::IpApi tmp_api = p->ptrs.ip_api; + int8_t num_layer = 0; + uint8_t tmp_next = p->get_ip_proto_next(); + bool outer_layer_only = (config->nestedIP == OUTER)? true: false; + bool outer_layer = false; + + while (layer::set_outer_ip_api(p, p->ptrs.ip_api, p->ip_proto_next, num_layer) && + tmp_api != p->ptrs.ip_api) + { + outer_layer = true; + + if(ReputationDecisionPerLayer(config, p, p->ptrs.ip_api, &decision_final)) + return decision_final; + + if(outer_layer_only) + { + p->ip_proto_next = tmp_next; + p->ptrs.ip_api = tmp_api; + return decision_final; + } + } + + p->ip_proto_next = tmp_next; + p->ptrs.ip_api = tmp_api; + + /*Check INNER IP, when configured or only one layer*/ + if (!outer_layer || (config->nestedIP == INNER) || (config->nestedIP == ALL)) + { + ReputationDecisionPerLayer(config, p, p->ptrs.ip_api, &decision_final); + } + + return (decision_final); +} + +static void snort_reputation(ReputationConfig* config, Packet* p) +{ + IPdecision decision; + + if (!config->iplist) + return; + + decision = ReputationDecision(config, p); + + if (DECISION_NULL == decision) + return; + + else if (BLACKLISTED == decision) + { + SnortEventqAdd(GID_REPUTATION, REPUTATION_EVENT_BLACKLIST); + Active::drop_packet(p, true); + // disable all preproc analysis and detection for this packet + DisableInspection(p); + p->disable_inspect = true; + if (p->flow) + { + p->flow->set_state(Flow::BLOCK); + p->flow->disable_inspection(); + } + + reputationstats.blacklisted++; + } + else if (MONITORED == decision) + { + SnortEventqAdd(GID_REPUTATION, REPUTATION_EVENT_MONITOR); + reputationstats.monitored++; + } + else if (WHITELISTED_TRUST == decision) + { + SnortEventqAdd(GID_REPUTATION, REPUTATION_EVENT_WHITELIST); + p->packet_flags |= PKT_IGNORE; + DisableInspection(p); + p->disable_inspect = true; + if (p->flow) + { + p->flow->set_state(Flow::ALLOW); + p->flow->disable_inspection(); + } + reputationstats.whitelisted++; + } +} + +//------------------------------------------------------------------------- +// class stuff +//------------------------------------------------------------------------- + +class Reputation : public Inspector +{ +public: + Reputation(ReputationConfig*); + ~Reputation(); + + void show(SnortConfig*) override; + void eval(Packet*) override; + +private: + ReputationConfig* config; +}; + +Reputation::Reputation(ReputationConfig* pc) +{ + config = pc; + reputationstats.memory_allocated = sfrt_flat_usage(config->iplist); +} + +Reputation::~Reputation() +{ + if ( config ) + { + delete config; + } +} + +void Reputation::show(SnortConfig*) +{ + PrintReputationConf(config); +} + +void Reputation::eval(Packet* p) +{ + Profile profile(reputationPerfStats); + + // precondition - what we registered for + assert(p->has_ip()); + + if (!p->is_rebuilt() && !IsReputationDisabled(p->flow)) + { + snort_reputation(config, p); + DisableReputation(p->flow); + ++reputationstats.packets; + } +} + +//------------------------------------------------------------------------- +// api stuff +//------------------------------------------------------------------------- + +static Module* mod_ctor() +{ return new ReputationModule; } + +static void mod_dtor(Module* m) +{ delete m; } + +static void reputation_init() +{ + ReputationFlowData::init(); +} + +static Inspector* reputation_ctor(Module* m) +{ + ReputationModule* mod = (ReputationModule*)m; + return new Reputation(mod->get_data()); +} + +static void reputation_dtor(Inspector* p) +{ + delete p; +} + +const InspectApi reputation_api = +{ + { + PT_INSPECTOR, + sizeof(InspectApi), + INSAPI_VERSION, + 0, + API_RESERVED, + API_OPTIONS, + REPUTATION_NAME, + REPUTATION_HELP, + mod_ctor, + mod_dtor + }, + IT_NETWORK, + (uint16_t)PktType::ANY_IP, + nullptr, // buffers + nullptr, // service + reputation_init, // pinit + nullptr, // pterm + nullptr, // tinit + nullptr, // tterm + reputation_ctor, + reputation_dtor, + nullptr, // ssn + nullptr // reset +}; + +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +{ + &reputation_api.base, + nullptr +}; +#else +const BaseApi* nin_reputation = &reputation_api.base; +#endif + diff --git a/src/network_inspectors/reputation/reputation_inspect.h b/src/network_inspectors/reputation/reputation_inspect.h new file mode 100644 index 000000000..99f6fffa4 --- /dev/null +++ b/src/network_inspectors/reputation/reputation_inspect.h @@ -0,0 +1,49 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +#ifndef REPUTATION_INSPECT_H +#define REPUTATION_INSPECT_H + +#include "protocols/packet.h" +#include "stream/stream_api.h" + +// Per-session data block containing current state +// of the Reputation preprocessor for the session. + +struct ReputationData +{ + bool disabled = false; +}; + +class ReputationFlowData : public FlowData +{ +public: + ReputationFlowData() : FlowData(flow_id){}; + + ~ReputationFlowData() { } + + static void init() + { flow_id = FlowData::get_flow_id(); } + +public: + static unsigned flow_id; + ReputationData session; +}; + +#endif + diff --git a/src/network_inspectors/reputation/reputation_module.cc b/src/network_inspectors/reputation/reputation_module.cc new file mode 100644 index 000000000..73625ea4f --- /dev/null +++ b/src/network_inspectors/reputation/reputation_module.cc @@ -0,0 +1,168 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// reputation_module.cc author Bhagya Tholpady + +#include "reputation_module.h" + +#include "utils/util.h" +#include +#include + +#include "reputation_parse.h" + +using namespace std; + +#define REPUTATION_EVENT_BLACKLIST_STR \ + "packets blacklisted" +#define REPUTATION_EVENT_WHITELIST_STR \ + "Packets whitelisted" +#define REPUTATION_EVENT_MONITOR_STR \ + "Packets monitored" + +static const Parameter s_params[] = +{ + { "blacklist", Parameter::PT_STRING, nullptr, nullptr, + "blacklist file name with ip lists" }, + + { "memcap", Parameter::PT_INT, "1:4095", "500", + "maximum total memory allocated" }, + + { "nested_ip", Parameter::PT_ENUM, "inner|outer|all", "inner", + "ip to use when there is IP encapsulation" }, + + { "priority", Parameter::PT_ENUM, "blacklist|whitelist", "whitelist", + "defines priority when there is a decision conflict during run-time" }, + + { "scan_local", Parameter::PT_BOOL, nullptr, "false", + "inspect local address defined in RFC 1918" }, + + { "white", Parameter::PT_ENUM, "unblack|trust", "unblack", + "specify the meaning of whitelist" }, + + { "whitelist", Parameter::PT_STRING, nullptr, nullptr, + "whitelist file name with ip lists" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const RuleMap reputation_rules[] = +{ + { REPUTATION_EVENT_BLACKLIST, REPUTATION_EVENT_BLACKLIST_STR }, + { REPUTATION_EVENT_WHITELIST, REPUTATION_EVENT_WHITELIST_STR }, + { REPUTATION_EVENT_MONITOR, REPUTATION_EVENT_MONITOR_STR }, + + { 0, nullptr } +}; + +//------------------------------------------------------------------------- +// reputation module +//------------------------------------------------------------------------- + +ReputationModule::ReputationModule() : Module(REPUTATION_NAME, REPUTATION_HELP, s_params) +{ + conf = nullptr; +} + +ReputationModule::~ReputationModule() +{ + if ( conf ) + { + delete conf; + } +} + +const RuleMap* ReputationModule::get_rules() const +{ return reputation_rules; } + +const PegInfo* ReputationModule::get_pegs() const +{ return reputation_peg_names; } + +PegCount* ReputationModule::get_counts() const +{ return (PegCount*)&reputationstats; } + +ProfileStats* ReputationModule::get_profile() const +{ return &reputationPerfStats; } + +bool ReputationModule::set(const char*, Value& v, SnortConfig*) +{ + if ( v.is("blacklist") ) + conf->blacklist_path = SnortStrdup(v.get_string()); + + else if ( v.is("memcap") ) + conf->memcap = v.get_long(); + + else if ( v.is("nested_ip") ) + conf->nestedIP = (NestedIP)v.get_long(); + + else if ( v.is("priority") ) + conf->priority = (IPdecision)(v.get_long() + 1); + + else if ( v.is("scan_local") ) + conf->scanlocal = v.get_bool(); + + else if ( v.is("white") ) + conf->whiteAction = (WhiteAction)v.get_long(); + + else if ( v.is("whitelist") ) + conf->whitelist_path = SnortStrdup(v.get_string()); + + else + return false; + + return true; +} + +ReputationConfig* ReputationModule::get_data() +{ + ReputationConfig* tmp = conf; + conf = nullptr; + return tmp; +} + +bool ReputationModule::begin(const char*, int, SnortConfig*) +{ + conf = new ReputationConfig; + + return true; +} + +bool ReputationModule::end(const char*, int, SnortConfig*) +{ + EstimateNumEntries(conf); + if (conf->numEntries <= 0) + { + LogMessage("WARNING: Can't find any whitelist/blacklist entries. " + "Reputation Preprocessor disabled.\n"); + return true; + } + + IpListInit(conf->numEntries + 1, conf); + + if ( (conf->priority == WHITELISTED_TRUST) && (conf->whiteAction == UNBLACK) ) + { + LogMessage("WARNING: Keyword \"whitelist\" for \"priority\" is not applied " + "when white action is unblack.\n"); + conf->priority = WHITELISTED_UNBLACK; + } + + LoadListFile(conf->blacklist_path, conf->local_black_ptr, conf); + LoadListFile(conf->whitelist_path, conf->local_white_ptr, conf); + return true; +} + diff --git a/src/network_inspectors/reputation/reputation_module.h b/src/network_inspectors/reputation/reputation_module.h new file mode 100644 index 000000000..8ebf7b110 --- /dev/null +++ b/src/network_inspectors/reputation/reputation_module.h @@ -0,0 +1,71 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// reputation_module.h author Bhagya Tholpady + +#ifndef REPUTATION_MODULE_H +#define REPUTATION_MODULE_H + +// Interface to the REPUTATION network inspector + +#include "framework/module.h" +#include "framework/bits.h" +#include "main/thread.h" +#include "reputation_config.h" + +#define GID_REPUTATION 136 + +#define REPUTATION_EVENT_BLACKLIST 1 +#define REPUTATION_EVENT_WHITELIST 2 +#define REPUTATION_EVENT_MONITOR 3 + +#define REPUTATION_NAME "reputation" +#define REPUTATION_HELP "reputation inspection" + +struct SnortConfig; + +extern THREAD_LOCAL ProfileStats reputationPerfStats; +extern unsigned long total_duplicates; +extern unsigned long total_invalids; + +class ReputationModule : public Module +{ +public: + ReputationModule(); + ~ReputationModule(); + + bool set(const char*, Value&, SnortConfig*) override; + bool begin(const char*, int, SnortConfig*) override; + bool end(const char*, int, SnortConfig*) override; + + unsigned get_gid() const override + { return GID_REPUTATION; } + + const RuleMap* get_rules() const override; + const PegInfo* get_pegs() const override; + PegCount* get_counts() const override; + ProfileStats* get_profile() const override; + + ReputationConfig* get_data(); + +private: + ReputationConfig* conf; +}; + +#endif + diff --git a/src/network_inspectors/reputation/reputation_parse.cc b/src/network_inspectors/reputation/reputation_parse.cc new file mode 100644 index 000000000..13947df24 --- /dev/null +++ b/src/network_inspectors/reputation/reputation_parse.cc @@ -0,0 +1,907 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// reputation_parse.cc author Hui Cao +// + +#include "reputation_parse.h" + +#include +#include +#include "parser/config_file.h" +#include "utils/util.h" +#include "main/snort_debug.h" + +using namespace std; + +enum +{ + IP_INSERT_SUCCESS = 0, + IP_INVALID, + IP_INSERT_FAILURE, + IP_INSERT_DUPLICATE, + IP_MEM_ALLOC_FAILURE +}; + +#define MAX_ADDR_LINE_LENGTH 8192 + +static char black_info[] = "blacklist"; +static char white_info[] = "whitelist"; +static char monitor_info[] = "monitorlist"; + +#define MAX_MSGS_TO_PRINT 20 + +unsigned long total_duplicates; +unsigned long total_invalids; + +int totalNumEntries = 0; + +ReputationConfig::~ReputationConfig() +{ + if (reputation_segment != nullptr) + free(reputation_segment); + + if (blacklist_path) + free(blacklist_path); + + if (whitelist_path) + free(whitelist_path); +} + + +uint32_t estimateSizeFromEntries(uint32_t num_entries, uint32_t memcap) +{ + uint64_t size; + uint64_t sizeFromEntries; + + /*memcap value is in Megabytes*/ + size = (uint64_t)memcap << 20; + + if (size > std::numeric_limits::max()) + size = std::numeric_limits::max(); + + /*Worst case, 15k ~ 2^14 per entry, plus one Megabytes for empty table*/ + if (num_entries > ((std::numeric_limits::max() - (1 << 20))>> 15)) + sizeFromEntries = std::numeric_limits::max(); + else + sizeFromEntries = (num_entries << 15) + (1 << 20); + + if (size > sizeFromEntries) + { + size = sizeFromEntries; + } + + return (uint32_t)size; +} + +void IpListInit(uint32_t maxEntries, ReputationConfig* config) +{ + uint8_t* base; + ListInfo* whiteInfo; + ListInfo* blackInfo; + MEM_OFFSET list_ptr; + + if (config->iplist == nullptr) + { + uint32_t mem_size; + mem_size = estimateSizeFromEntries(maxEntries, config->memcap); + config->reputation_segment = (uint8_t*)malloc(mem_size); + if (config->reputation_segment == nullptr) + { + FatalError("Failed to allocate memory for local segment\n"); + } + + segment_meminit(config->reputation_segment, mem_size); + base = config->reputation_segment; + + /*DIR_16x7_4x4 for performance, but memory usage is high + *Use DIR_8x16 worst case IPV4 5K, IPV6 15K (bytes) + *Use DIR_16x7_4x4 worst case IPV4 500, IPV6 2.5M + */ + config->iplist = sfrt_flat_new(DIR_8x16, IPv6, maxEntries, config->memcap); + if (config->iplist == nullptr) + { + FatalError("Failed to create IP list.\n"); + } + + list_ptr = segment_calloc((size_t)DECISION_MAX, sizeof(ListInfo)); + config->iplist->list_info = list_ptr; + + config->local_black_ptr = list_ptr + BLACKLISTED * sizeof(ListInfo); + blackInfo = (ListInfo*)&base[config->local_black_ptr]; + blackInfo->listType = BLACKLISTED; + blackInfo->listIndex = BLACKLISTED + 1; + if (UNBLACK == config->whiteAction) + { + config->local_white_ptr = list_ptr + WHITELISTED_UNBLACK * sizeof(ListInfo); + whiteInfo = (ListInfo*)&base[config->local_white_ptr]; + whiteInfo->listType = WHITELISTED_UNBLACK; + whiteInfo->listIndex = WHITELISTED_UNBLACK + 1; + } + else + { + config->local_white_ptr = list_ptr + WHITELISTED_TRUST * sizeof(ListInfo); + whiteInfo = (ListInfo*)&base[config->local_white_ptr]; + whiteInfo->listType = WHITELISTED_TRUST; + whiteInfo->listIndex = WHITELISTED_TRUST + 1; + } + } +} + +static inline IPrepInfo* getLastIndex(IPrepInfo* repInfo, uint8_t* base, int* lastIndex) +{ + int i; + + assert(repInfo); + + /* Move to the end of current info*/ + while (repInfo->next) + { + repInfo = (IPrepInfo*)&base[repInfo->next]; + } + + for (i = 0; i < NUM_INDEX_PER_ENTRY; i++) + { + if (!repInfo->listIndexes[i]) + break; + } + + if (i > 0) + { + *lastIndex = i-1; + return repInfo; + } + else + { + return nullptr; + } +} + +static inline int duplicateInfo(IPrepInfo* destInfo,IPrepInfo* currentInfo, + uint8_t* base) +{ + int bytesAllocated = 0; + + while (currentInfo) + { + INFO nextInfo; + *destInfo = *currentInfo; + if (!currentInfo->next) + break; + nextInfo = segment_calloc(1,sizeof(IPrepInfo)); + if (!nextInfo) + { + destInfo->next = 0; + return -1; + } + else + { + destInfo->next = nextInfo; + } + bytesAllocated += sizeof(IPrepInfo); + currentInfo = (IPrepInfo*)&base[currentInfo->next]; + destInfo = (IPrepInfo*)&base[nextInfo]; + } + + return bytesAllocated; +} + +static int64_t updateEntryInfo(INFO* current, INFO new_entry, SaveDest saveDest, uint8_t* base) +{ + IPrepInfo* currentInfo; + IPrepInfo* newInfo; + IPrepInfo* destInfo; + IPrepInfo* lastInfo; + int64_t bytesAllocated = 0; + int i; + char newIndex; + + if (!(*current)) + { + /* Copy the data to segment memory*/ + *current = segment_calloc(1,sizeof(IPrepInfo)); + if (!(*current)) + { + return -1; + } + bytesAllocated = sizeof(IPrepInfo); + } + + if (*current == new_entry) + return bytesAllocated; + + currentInfo = (IPrepInfo*)&base[*current]; + newInfo = (IPrepInfo*)&base[new_entry]; + + /*The latest information is always the last entry + */ + lastInfo = getLastIndex(newInfo, base, &i); + + if (!lastInfo) + { + return bytesAllocated; + } + newIndex = lastInfo->listIndexes[i++]; + + DEBUG_WRAP(DebugMessage(DEBUG_REPUTATION, "Current IP reputation information: \n"); ); + DEBUG_WRAP(ReputationPrintRepInfo(currentInfo, base); ); + DEBUG_WRAP(DebugMessage(DEBUG_REPUTATION, "New IP reputation information: \n"); ); + DEBUG_WRAP(ReputationPrintRepInfo(newInfo, base); ); + + if (SAVE_TO_NEW == saveDest) + { + int bytesDuplicated; + + /* When updating new entry, current information should be reserved + * because current information is inherited from parent + */ + if ((bytesDuplicated = duplicateInfo(newInfo, currentInfo, base)) < 0) + return -1; + else + bytesAllocated += bytesDuplicated; + + destInfo = newInfo; + } + else + { + destInfo = currentInfo; + } + + /* Add the new list information to the end + * This way, the order of list information is preserved. + * The first one always has the highest priority, + * because it is checked first during lookup. + */ + + while (destInfo->next) + { + destInfo = (IPrepInfo*)&base[destInfo->next]; + } + + for (i = 0; i < NUM_INDEX_PER_ENTRY; i++) + { + if (!destInfo->listIndexes[i]) + break; + else if (destInfo->listIndexes[i] == newIndex) + { + DEBUG_WRAP(DebugMessage(DEBUG_REPUTATION, "Final IP reputation information: \n"); ); + DEBUG_WRAP(ReputationPrintRepInfo(destInfo, base); ); + return bytesAllocated; + } + } + + if (i < NUM_INDEX_PER_ENTRY) + { + destInfo->listIndexes[i] = newIndex; + } + else + { + IPrepInfo* nextInfo; + MEM_OFFSET ipInfo_ptr = segment_calloc(1,sizeof(IPrepInfo)); + if (!ipInfo_ptr) + return -1; + destInfo->next = ipInfo_ptr; + nextInfo = (IPrepInfo*)&base[destInfo->next]; + nextInfo->listIndexes[0] = newIndex; + bytesAllocated += sizeof(IPrepInfo); + } + + DEBUG_WRAP(DebugMessage(DEBUG_REPUTATION, "Final IP reputation information: \n"); ); + DEBUG_WRAP(ReputationPrintRepInfo(destInfo, base); ); + + return bytesAllocated; +} + +static int AddIPtoList(sfip_t* ipAddr,INFO ipInfo_ptr, ReputationConfig* config) +{ + int iRet; + int iFinalRet = IP_INSERT_SUCCESS; + /*This variable is used to check whether a more generic address + * overrides specific address + */ + uint32_t usageBeforeAdd; + uint32_t usageAfterAdd; + + if (ipAddr->family == AF_INET) + { + ipAddr->ip32[0] = ntohl(ipAddr->ip32[0]); + } + else if (ipAddr->family == AF_INET6) + { + int i; + for (i = 0; i < 4; i++) + ipAddr->ip32[i] = ntohl(ipAddr->ip32[i]); + } + +#ifdef DEBUG_MSGS + if (nullptr != sfrt_flat_lookup((void*)ipAddr, config->iplist)) + { + DebugFormat(DEBUG_REPUTATION, "Find address before insert: %s\n", sfip_to_str(ipAddr) ); + } + else + { + DebugFormat(DEBUG_REPUTATION, + "Can't find address before insert: %s\n", sfip_to_str(ipAddr) ); + } +#endif + + usageBeforeAdd = sfrt_flat_usage(config->iplist); + + /*Check whether the same or more generic address is already in the table*/ + if (nullptr != sfrt_flat_lookup((void*)ipAddr, config->iplist)) + { + iFinalRet = IP_INSERT_DUPLICATE; + } + + iRet = sfrt_flat_insert((void*)ipAddr, (unsigned char)ipAddr->bits, ipInfo_ptr, RT_FAVOR_ALL, + config->iplist, &updateEntryInfo); + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Unused memory: %d \n",segment_unusedmem()); ); + + if (RT_SUCCESS == iRet) + { +#ifdef DEBUG_MSGS + IPrepInfo* result; + DebugFormat(DEBUG_REPUTATION, "Number of entries input: %d, in table: %d \n", + totalNumEntries,sfrt_flat_num_entries(config->iplist) ); + DebugFormat(DEBUG_REPUTATION, "Memory allocated: %d \n",sfrt_flat_usage(config->iplist) ); + result = (IPrepInfo*)sfrt_flat_lookup((void*)ipAddr, config->iplist); + if (nullptr != result) + { + DebugFormat(DEBUG_REPUTATION, "Find address after insert: %s \n",sfip_to_str(ipAddr) ); + DEBUG_WRAP(ReputationPrintRepInfo(result, (uint8_t*)config->iplist); ); + } +#endif + totalNumEntries++; + } + else if (MEM_ALLOC_FAILURE == iRet) + { + iFinalRet = IP_MEM_ALLOC_FAILURE; + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Insert error: %d for address: %s \n",iRet, + sfip_to_str(ipAddr) ); ); + } + else + { + iFinalRet = IP_INSERT_FAILURE; + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Insert error: %d for address: %s \n",iRet, + sfip_to_str(ipAddr) ); ); + } + + usageAfterAdd = sfrt_flat_usage(config->iplist); + /*Compare in the same scale*/ + if (usageAfterAdd > (config->memcap << 20)) + { + iFinalRet = IP_MEM_ALLOC_FAILURE; + } + /*Check whether there a more specific address will be overridden*/ + if (usageBeforeAdd > usageAfterAdd ) + { + iFinalRet = IP_INSERT_DUPLICATE; + } + + return iFinalRet; +} + +static int snort_pton__address(char const* src, sfip_t* dest) +{ + unsigned char _temp[sizeof(struct in6_addr)]; + + if ( inet_pton(AF_INET, src, _temp) == 1 ) + { + dest->family = AF_INET; + dest->bits = 32; + } + else if ( inet_pton(AF_INET6, src, _temp) == 1 ) + { + dest->family = AF_INET6; + dest->bits = 128; + } + else + { + return 0; + } + + memcpy(&dest->ip8[0], _temp, sizeof(_temp)); + + return 1; +} + +#define isident(x) (isxdigit((x)) || (x) == ':' || (x) == '.') +static int snort_pton(char const* src, sfip_t* dest) +{ + char ipbuf[INET6_ADDRSTRLEN]; + char cidrbuf[sizeof("128")]; + char* out; + enum + { + BEGIN, IP, CIDR1, CIDR2, END, INVALID + } state; + + memset(ipbuf, '\0', sizeof(ipbuf)); + memset(cidrbuf, '\0', sizeof(cidrbuf)); + + state = BEGIN; + + while ( *src ) + { + char ch = *src; + + //printf("State:%d; C:%x; P:%p\n", state, ch, src ); + src += 1; + + switch ( state ) + { + // Scan for beginning of IP address + case BEGIN: + if ( isident((int)ch) ) + { + // Set the first ipbuff byte and change state + out = ipbuf; + *out++ = ch; + state = IP; + } + else if ( !isspace((int)ch) ) + { + state = INVALID; + } + break; + + // Fill in ipbuf with ip identifier characters + // Move to CIDR1 if a cidr divider (i.e., '/') is found. + case IP: + if ( isident((int)ch) && (out - ipbuf + 1) < (int)sizeof(ipbuf) ) + { + *out++ = ch; + } + else if ( ch == '/' ) + { + state = CIDR1; + } + else if ( isspace((int)ch) ) + { + state = END; + } + else + { + state = INVALID; + } + break; + + // First cidr digit + case CIDR1: + if ( !isdigit((int)ch) ) + { + state = INVALID; + } + else + { + // Set output to the cidrbuf buffer + out = cidrbuf; + *out++ = ch; + state = CIDR2; + } + break; + + // Consume any addition digits for cidrbuf + case CIDR2: + if ( isdigit((int)ch) && (out - cidrbuf + 1) < (int)sizeof(cidrbuf) ) + { + *out++ = ch; + } + else if ( isspace((int)ch) ) + { + state = END; + } + else + { + state = INVALID; + } + break; + + // Scan for junk at the EOL + case END: + if ( !isspace((int)ch) ) + { + state = INVALID; + } + break; + + // Can't get here + default: + break; + } + + if ( state == INVALID ) + return -1; + } + + if ( snort_pton__address(ipbuf, dest) < 1 ) + return 0; + + if ( *cidrbuf ) + { + char* end; + int value = strtol(cidrbuf, &end, 10); + + if ( value > dest->bits || value <= 0 || errno == ERANGE ) + return 0; + + dest->bits = value; + } + + return 1; +} + +static int ProcessLine(char* line, INFO info, ReputationConfig* config) +{ + sfip_t address; + + if ( !line || *line == '\0' ) + return IP_INSERT_SUCCESS; + + if ( snort_pton(line, &address) < 1 ) + return IP_INVALID; + + return AddIPtoList(&address, info, config); +} + +static int UpdatePathToFile(char* full_path_filename, unsigned int max_size, char* filename) +{ + const char* snort_conf_dir = get_snort_conf_dir(); + + if (!snort_conf_dir || !(*snort_conf_dir) || !full_path_filename || !filename) + { + FatalError("can't create path.\n"); + return 0; + } + /*filename is too long*/ + if ( max_size < strlen(filename) ) + { + FatalError("The file name length %u is longer than allowed %u.\n", (unsigned)strlen( + filename), max_size); + return 0; + } + + /* + * If an absolute path is specified, then use that. + */ +#ifndef WIN32 + if (filename[0] == '/') + { + snprintf(full_path_filename, max_size, "%s", filename); + } + else + { + /* + * Set up the file name directory. + */ + if (snort_conf_dir[strlen(snort_conf_dir) - 1] == '/') + { + snprintf(full_path_filename,max_size, + "%s%s", snort_conf_dir, filename); + } + else + { + snprintf(full_path_filename, max_size, + "%s/%s", snort_conf_dir, filename); + } + } +#else + if (strlen(filename)>3 && filename[1]==':' && filename[2]=='\\') + { + snprintf(full_path_filename, max_size, "%s", filename); + } + else + { + /* + ** Set up the file name directory + */ + if (snort_conf_dir[strlen(snort_conf_dir) - 1] == '\\' || + snort_conf_dir[strlen(snort_conf_dir) - 1] == '/' ) + { + snprintf(full_path_filename,max_size, + "%s%s", snort_conf_dir, filename); + } + else + { + snprintf(full_path_filename, max_size, + "%s\\%s", snort_conf_dir, filename); + } + } +#endif + return 1; +} + +static char* GetListInfo(INFO info) +{ + uint8_t* base; + ListInfo* info_value; + base = (uint8_t*)segment_basePtr(); + info_value = (ListInfo*)(&base[info]); + if (!info_value) + return nullptr; + switch (info_value->listType) + { + case DECISION_NULL: + return nullptr; + break; + case BLACKLISTED: + return black_info; + break; + case WHITELISTED_UNBLACK: + return white_info; + break; + case MONITORED: + return monitor_info; + break; + case WHITELISTED_TRUST: + return white_info; + break; + default: + return nullptr; + } + return nullptr; +} + +void LoadListFile(char* filename, INFO info, ReputationConfig* config) +{ + char linebuf[MAX_ADDR_LINE_LENGTH]; + char full_path_filename[PATH_MAX+1]; + int addrline = 0; + FILE* fp = nullptr; + char* cmt = nullptr; + char* list_info; + ListInfo* listInfo; + IPrepInfo* ipInfo; + MEM_OFFSET ipInfo_ptr; + uint8_t* base; + + /*entries processing statistics*/ + unsigned int duplicate_count = 0; /*number of duplicates in this file*/ + unsigned int invalid_count = 0; /*number of invalid entries in this file*/ + unsigned int fail_count = 0; /*number of invalid entries in this file*/ + unsigned int num_loaded_before = 0; /*number of valid entries loaded */ + + if ((nullptr == filename)||(0 == info)|| (nullptr == config)||config->memCapReached) + return; + + UpdatePathToFile(full_path_filename, PATH_MAX, filename); + + list_info = GetListInfo(info); + + if (!list_info) + return; + + /*convert list info to ip entry info*/ + ipInfo_ptr = segment_calloc(1,sizeof(IPrepInfo)); + if (!(ipInfo_ptr)) + { + return; + } + base = (uint8_t*)config->iplist; + ipInfo = ((IPrepInfo*)&base[ipInfo_ptr]); + listInfo = ((ListInfo*)&base[info]); + ipInfo->listIndexes[0] = listInfo->listIndex; + + LogMessage(" Processing %s file %s\n", list_info, full_path_filename); + + if ((fp = fopen(full_path_filename, "r")) == nullptr) + { + char errBuf[STD_BUF]; +#ifdef WIN32 + snprintf(errBuf, STD_BUF, "%s", strerror(errno)); +#else + strerror_r(errno, errBuf, STD_BUF); +#endif + errBuf[STD_BUF-1] = '\0'; + ErrorMessage("Unable to open address file %s, Error: %s\n", full_path_filename, errBuf); + return; + } + + num_loaded_before = sfrt_flat_num_entries(config->iplist); + while ( fgets(linebuf, MAX_ADDR_LINE_LENGTH, fp) ) + { + int iRet; + addrline++; + + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Reputation configurations: %s\n",linebuf); ); + + // Remove comments + if ( (cmt = strchr(linebuf, '#')) ) + *cmt = '\0'; + + // Remove newline as well, prevent double newline in logging. + if ( (cmt = strchr(linebuf, '\n')) ) + *cmt = '\0'; + + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Reputation configurations: %s\n",linebuf); ); + + /* process the line */ + iRet = ProcessLine(linebuf, ipInfo_ptr, config); + + if (IP_INSERT_SUCCESS == iRet) + { + continue; + } + else if (IP_INSERT_FAILURE == iRet && fail_count++ < MAX_MSGS_TO_PRINT) + { + ErrorMessage(" (%d) => Failed to insert address: \'%s\'\n", addrline, linebuf); + } + else if (IP_INVALID == iRet && invalid_count++ < MAX_MSGS_TO_PRINT) + { + ErrorMessage(" (%d) => Invalid address: \'%s\'\n", addrline, linebuf); + } + else if (IP_INSERT_DUPLICATE == iRet && duplicate_count++ < MAX_MSGS_TO_PRINT) + { + ErrorMessage(" (%d) => Re-defined address: '%s'\n", addrline, linebuf); + } + else if (IP_MEM_ALLOC_FAILURE == iRet) + { + ErrorMessage( + "WARNING: %s(%d) => Memcap %u Mbytes reached when inserting IP Address: %s\n", + full_path_filename, addrline, config->memcap,linebuf); + + config->memCapReached = true; + break; + } + } + + total_duplicates += duplicate_count; + total_invalids += invalid_count; + /*Print out the summary*/ + if (fail_count > MAX_MSGS_TO_PRINT) + ErrorMessage(" Additional addresses failed insertion but were not listed.\n"); + if (invalid_count > MAX_MSGS_TO_PRINT) + ErrorMessage(" Additional invalid addresses were not listed.\n"); + if (duplicate_count > MAX_MSGS_TO_PRINT) + ErrorMessage(" Additional duplicate addresses were not listed.\n"); + + LogMessage(" Reputation entries loaded: %u, invalid: %u, re-defined: %u (from file %s)\n", + sfrt_flat_num_entries(config->iplist) - num_loaded_before, + invalid_count, duplicate_count, full_path_filename); + + fclose(fp); +} + +int numLinesInFile(char* fname) +{ + FILE* fp; + uint32_t numlines = 0; + char buf[MAX_ADDR_LINE_LENGTH]; + + fp = fopen(fname, "rb"); + + if (nullptr == fp) + return 0; + + while ((fgets(buf, MAX_ADDR_LINE_LENGTH, fp)) != nullptr) + { + if (buf[0] != '#') + { + numlines++; + if (numlines == std::numeric_limits::max()) + { + fclose(fp); + return std::numeric_limits::max(); + } + } + } + + fclose(fp); + return numlines; +} + +static int LoadFile(int totalLines, char* path) +{ + int numlines; + char full_path_filename[PATH_MAX+1]; + + if (!path) + return 0; + + errno = 0; + UpdatePathToFile(full_path_filename,PATH_MAX, path); + numlines = numLinesInFile(full_path_filename); + + if ((0 == numlines) && (0 != errno)) + { + char errBuf[STD_BUF]; +#ifdef WIN32 + snprintf(errBuf, STD_BUF, "%s", strerror(errno)); +#else + strerror_r(errno, errBuf, STD_BUF); +#endif + FatalError("Unable to open address file %s, Error: %s\n", full_path_filename, errBuf); + } + + if (totalLines + numlines < totalLines) + { + FatalError("Too many entries in one file.\n"); + } + + return numlines; +} + +void EstimateNumEntries(ReputationConfig* config) +{ + int totalLines = 0; + + totalLines += LoadFile(totalLines, config->blacklist_path); + totalLines += LoadFile(totalLines, config->whitelist_path); + + config->numEntries = totalLines; +} + +void ReputationRepInfo(IPrepInfo* repInfo, uint8_t* base, char* repInfoBuff, + int bufLen) +{ + char* index = repInfoBuff; + int len = bufLen -1; + int writed; + + writed = snprintf(index, len, "Reputation Info: "); + if (writed >= len || writed < 0) + return; + + index += writed; + len -= writed; + + while (repInfo) + { + int i; + for (i = 0; i < NUM_INDEX_PER_ENTRY; i++) + { + writed = snprintf(index, len, "%d,",repInfo->listIndexes[i]); + if (writed >= len || writed < 0) + return; + else + { + index += writed; + len -=writed; + } + } + writed = snprintf(index, len, "->"); + if (writed >= len || writed < 0) + return; + else + { + index += writed; + len -=writed; + } + + if (!repInfo->next) + break; + + repInfo = (IPrepInfo*)(&base[repInfo->next]); + } +} + +#ifdef DEBUG_MSGS +void ReputationPrintRepInfo(IPrepInfo* repInfo, uint8_t* base) +{ + char repInfoBuff[STD_BUF]; + int len = STD_BUF -1; + + repInfoBuff[STD_BUF -1] = '\0'; + + ReputationRepInfo(repInfo, base, repInfoBuff, len); + + DEBUG_WRAP(DebugFormat(DEBUG_REPUTATION, "Reputation Info: %s \n", + repInfoBuff); ); +} + +#endif + diff --git a/src/network_inspectors/reputation/reputation_parse.h b/src/network_inspectors/reputation/reputation_parse.h new file mode 100644 index 000000000..e33f3ce31 --- /dev/null +++ b/src/network_inspectors/reputation/reputation_parse.h @@ -0,0 +1,29 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. +// Copyright (C) 2004-2013 Sourcefire, Inc. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +#ifndef REPUTATION_PARSE_H +#define REPUTATION_PARSE_H + +#include "reputation_config.h" + +void IpListInit(uint32_t,ReputationConfig *config); +void EstimateNumEntries(ReputationConfig* config); +void LoadListFile(char* filename, INFO info, ReputationConfig* config); + +#endif diff --git a/src/protocols/packet.h b/src/protocols/packet.h index 28c761562..a83b7155a 100644 --- a/src/protocols/packet.h +++ b/src/protocols/packet.h @@ -134,6 +134,7 @@ struct SO_PUBLIC Packet uint8_t num_layers; /* index into layers for next encap */ uint8_t ip_proto_next; /* the protocol ID after IP and all IP6 extension */ + bool disable_inspect; // nothing after this point is zeroed ... diff --git a/tools/snort2lua/preprocessor_states/CMakeLists.txt b/tools/snort2lua/preprocessor_states/CMakeLists.txt index 5f54037ba..d1c9fe199 100644 --- a/tools/snort2lua/preprocessor_states/CMakeLists.txt +++ b/tools/snort2lua/preprocessor_states/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(preprocessor_states pps_http_inspect_server.cc pps_normalizers.cc pps_perfmonitor.cc + pps_reputation.cc pps_rpc_decode.cc pps_sip.cc pps_ssh.cc diff --git a/tools/snort2lua/preprocessor_states/Makefile.am b/tools/snort2lua/preprocessor_states/Makefile.am index 3f56e00e3..2d1348341 100644 --- a/tools/snort2lua/preprocessor_states/Makefile.am +++ b/tools/snort2lua/preprocessor_states/Makefile.am @@ -14,6 +14,7 @@ pps_http_inspect.cc \ pps_http_inspect_server.cc \ pps_normalizers.cc \ pps_perfmonitor.cc \ +pps_reputation.cc \ pps_rpc_decode.cc \ pps_sip.cc \ pps_ssh.cc \ diff --git a/tools/snort2lua/preprocessor_states/pps_reputation.cc b/tools/snort2lua/preprocessor_states/pps_reputation.cc new file mode 100644 index 000000000..2ddae29d0 --- /dev/null +++ b/tools/snort2lua/preprocessor_states/pps_reputation.cc @@ -0,0 +1,172 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// pps_reputation.cc author Bhagya Tholpady + +#include +#include + +#include "conversion_state.h" +#include "helpers/s2l_util.h" + +namespace preprocessors +{ +namespace +{ +class Reputation : public ConversionState +{ +public: + Reputation(Converter& c) : ConversionState(c) { } + virtual ~Reputation() { } + virtual bool convert(std::istringstream& data_stream); +}; +} // namespace + +bool Reputation::convert(std::istringstream& data_stream) +{ + std::string keyword; + bool retval = true; + + table_api.open_table("reputation"); + + // parse the file configuration + while (util::get_string(data_stream, keyword, ",")) + { + bool tmpval = true; + std::istringstream arg_stream(keyword); + + // should be gauranteed to happen. Checking for error just cause + if (!(arg_stream >> keyword)) + tmpval = false; + + else if (!keyword.compare("shared_mem")) + table_api.add_deleted_comment("shared_mem"); + + else if (!keyword.compare("shared_refresh")) + table_api.add_deleted_comment("shared_refresh"); + + else if (!keyword.compare("blacklist")) + { + std::string file_name; + if( arg_stream >> file_name) + { + tmpval = table_api.add_option("blacklist", file_name); + } + else + { + data_api.failed_conversion(arg_stream, "reputation: blacklist "); + tmpval = false; + } + } + else if (!keyword.compare("memcap")) + { + tmpval = parse_int_option("memcap", arg_stream, false); + } + else if (!keyword.compare("nested_ip")) + { + std::string val; + if (!(arg_stream >> val)) + data_api.failed_conversion(arg_stream, "reputation: nested_ip "); + else if (!val.compare("inner")) + table_api.add_option("nested_ip", "inner"); + else if (!val.compare("outer")) + table_api.add_option("nested_ip", "outer"); + else if (!val.compare("both")) + table_api.add_option("nested_ip", "all"); + else + { + data_api.failed_conversion(arg_stream, "reputation: nested_ip " + val); + } + } + else if (!keyword.compare("priority")) + { + std::string val; + if (!(arg_stream >> val)) + data_api.failed_conversion(arg_stream, "reputation: priority "); + else if (!val.compare("whitelist")) + table_api.add_option("priority", "whitelist"); + else if (!val.compare("blacklist")) + table_api.add_option("priority", "blacklist"); + else + { + data_api.failed_conversion(arg_stream, "reputation: priority " + val); + } + } + else if (!keyword.compare("scan_local")) + { + tmpval = table_api.add_option("scan_local", true); + } + else if (!keyword.compare("white")) + { + std::string val; + if (!(arg_stream >> val)) + data_api.failed_conversion(arg_stream, "reputation: white "); + else if (!val.compare("unblack")) + table_api.add_option("white", "unblack"); + else if (!val.compare("trust")) + table_api.add_option("white", "trust"); + else + { + data_api.failed_conversion(arg_stream, "reputation: white " + val); + } + } + else if (!keyword.compare("whitelist")) + { + std::string file_name; + if( arg_stream >> file_name) + { + tmpval = table_api.add_option("whitelist", file_name); + } + else + { + data_api.failed_conversion(arg_stream, "reputation: whitelist "); + tmpval = false; + } + } + else + { + tmpval = false; + } + + if (!tmpval) + { + data_api.failed_conversion(arg_stream, keyword); + retval = false; + } + } + + return retval; +} + +/************************** + ******* A P I *********** + **************************/ + +static ConversionState* ctor(Converter& c) +{ + return new Reputation(c); +} + +static const ConvertMap preprocessor_reputation = +{ + "reputation", + ctor, +}; + +const ConvertMap* reputation_map = &preprocessor_reputation; +} + diff --git a/tools/snort2lua/preprocessor_states/preprocessor_api.cc b/tools/snort2lua/preprocessor_states/preprocessor_api.cc index c8d91f83a..601bc9915 100644 --- a/tools/snort2lua/preprocessor_states/preprocessor_api.cc +++ b/tools/snort2lua/preprocessor_states/preprocessor_api.cc @@ -38,6 +38,7 @@ extern const ConvertMap* normalizer_ip4_map; extern const ConvertMap* normalizer_ip6_map; extern const ConvertMap* normalizer_tcp_map; extern const ConvertMap* perfmonitor_map; +extern const ConvertMap* reputation_map; extern const ConvertMap* rpc_decode_map; extern const ConvertMap* sip_map; extern const ConvertMap* ssh_map; @@ -72,6 +73,7 @@ const std::vector preprocessor_api = normalizer_ip6_map, normalizer_tcp_map, perfmonitor_map, + reputation_map, rpc_decode_map, sip_map, ssh_map,