From: Jonathan McDowell Date: Mon, 4 Jun 2018 14:41:41 +0000 (+0100) Subject: Scratch handlers for SnortState (#48) X-Git-Tag: 3.0.0-246~72 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=6b3ff896d5ef686331badbd73e34d13e86d7c406;p=thirdparty%2Fsnort3.git Scratch handlers for SnortState (#48) * Add basic infrastructure for registering SnortState scratch space * Port ips_sd_pattern to new scratch space registration * Port ips_pcre to new scratch space registration * Replace SnortState with a std::vector directly All of the custom fields in SnortState have been moved over to the scratch space registration method and all that's left is the scratch vector. Remove the SnortState structure and just use a vector directly within SnortConfig. * Enable dynamic building of ips_{pcre,regex,sd_pattern} + Hyperscan MPSE Now that these modules dynamic request space within the SnortConfig state there's no need for them to be statically linked into the binary. * Port ips_regex to new scratch space registration * Port hyperscan mpse to new scratch space registration --- diff --git a/src/ips_options/CMakeLists.txt b/src/ips_options/CMakeLists.txt index b25f5a60b..2ee6946be 100644 --- a/src/ips_options/CMakeLists.txt +++ b/src/ips_options/CMakeLists.txt @@ -27,6 +27,7 @@ SET( PLUGIN_LIST ips_isdataat.cc ips_itype.cc ips_msg.cc + ips_pcre.cc ips_priority.cc ips_raw_data.cc ips_rem.cc @@ -61,8 +62,6 @@ set (IPS_SOURCES ips_metadata.cc ips_options.cc ips_options.h - ips_pcre.cc - ips_pcre.h ips_pkt_data.cc ips_reference.cc ips_replace.cc @@ -71,8 +70,10 @@ set (IPS_SOURCES ) if ( HAVE_HYPERSCAN ) - set(OPTION_LIST ips_regex.cc ips_regex.h - ips_sd_pattern.cc ips_sd_pattern.h + set(PLUGIN_LIST + ${PLUGIN_LIST} + ips_regex.cc + ips_sd_pattern.cc sd_credit_card.cc sd_credit_card.h) endif () @@ -114,6 +115,7 @@ else (STATIC_IPS_OPTIONS) add_dynamic_module(ips_isdataat ips_options ips_isdataat.cc) add_dynamic_module(ips_itype ips_options ips_itype.cc) add_dynamic_module(ips_msg ips_options ips_msg.cc) + add_dynamic_module(ips_pcre ips_options ips_pcre.cc) add_dynamic_module(ips_priority ips_options ips_priority.cc) add_dynamic_module(ips_raw_data ips_options ips_raw_data.cc) add_dynamic_module(ips_rem ips_options ips_rem.cc) @@ -128,6 +130,10 @@ else (STATIC_IPS_OPTIONS) add_dynamic_module(ips_tos ips_options ips_tos.cc) add_dynamic_module(ips_ttl ips_options ips_ttl.cc) add_dynamic_module(ips_window ips_options ips_window.cc) +if ( HAVE_HYPERSCAN ) + add_dynamic_module(ips_regex ips_options ips_regex.cc) + add_dynamic_module(ips_sd_pattern ips_options ips_sd_pattern.cc sd_credit_card.cc sd_credit_card.h) +endif ( HAVE_HYPERSCAN ) endif (STATIC_IPS_OPTIONS) diff --git a/src/ips_options/ips_options.cc b/src/ips_options/ips_options.cc index 2bcc0ffe3..7edde2701 100644 --- a/src/ips_options/ips_options.cc +++ b/src/ips_options/ips_options.cc @@ -36,13 +36,8 @@ extern const BaseApi* ips_flow; extern const BaseApi* ips_flowbits; extern const BaseApi* ips_md5; extern const BaseApi* ips_metadata; -extern const BaseApi* ips_pcre; extern const BaseApi* ips_pkt_data; extern const BaseApi* ips_reference; -#ifdef HAVE_HYPERSCAN -extern const BaseApi* ips_regex; -extern const BaseApi* ips_sd_pattern; -#endif extern const BaseApi* ips_replace; extern const BaseApi* ips_service; extern const BaseApi* ips_sha256; @@ -72,6 +67,7 @@ extern const BaseApi* ips_ip_proto[]; extern const BaseApi* ips_isdataat[]; extern const BaseApi* ips_itype[]; extern const BaseApi* ips_msg[]; +extern const BaseApi* ips_pcre[]; extern const BaseApi* ips_priority[]; extern const BaseApi* ips_raw_data[]; extern const BaseApi* ips_rem[]; @@ -87,6 +83,10 @@ extern const BaseApi* ips_tos[]; extern const BaseApi* ips_ttl[]; extern const BaseApi* ips_bufferlen[]; extern const BaseApi* ips_window[]; +#ifdef HAVE_HYPERSCAN +extern const BaseApi* ips_regex[]; +extern const BaseApi* ips_sd_pattern[]; +#endif #endif static const BaseApi* ips_options[] = @@ -100,13 +100,8 @@ static const BaseApi* ips_options[] = ips_flowbits, ips_md5, ips_metadata, - ips_pcre, ips_pkt_data, ips_reference, -#ifdef HAVE_HYPERSCAN - ips_regex, - ips_sd_pattern, -#endif ips_replace, ips_service, ips_sha256, @@ -142,6 +137,7 @@ void load_ips_options() PluginManager::load_plugins(ips_isdataat); PluginManager::load_plugins(ips_itype); PluginManager::load_plugins(ips_msg); + PluginManager::load_plugins(ips_pcre); PluginManager::load_plugins(ips_priority); PluginManager::load_plugins(ips_raw_data); PluginManager::load_plugins(ips_rem); @@ -157,6 +153,10 @@ void load_ips_options() PluginManager::load_plugins(ips_ttl); PluginManager::load_plugins(ips_bufferlen); PluginManager::load_plugins(ips_window); +#ifdef HAVE_HYPERSCAN + PluginManager::load_plugins(ips_regex); + PluginManager::load_plugins(ips_sd_pattern); +#endif #endif } diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index cef9dd67e..238d717e2 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -23,8 +23,6 @@ #include "config.h" #endif -#include "ips_pcre.h" - #include #include @@ -88,6 +86,8 @@ static int s_ovector_max = 0; // by verify; search uses the value in snort conf static int s_ovector_size = 0; +static unsigned scratch_index; + static THREAD_LOCAL ProfileStats pcrePerfStats; //------------------------------------------------------------------------- @@ -363,8 +363,8 @@ static bool pcre_search( found_offset = -1; - SnortState* ss = SnortConfig::get_conf()->state + get_instance_id(); - assert(ss->pcre_ovector); + std::vector ss = SnortConfig::get_conf()->state[get_instance_id()]; + assert(ss[scratch_index]); int result = pcre_exec( pcre_data->re, /* result of pcre_compile() */ @@ -373,7 +373,7 @@ static bool pcre_search( len, /* the length of the subject string */ start_offset, /* start at offset 0 in the subject */ 0, /* options(handled at compile time */ - ss->pcre_ovector, /* vector for substring information */ + (int*)ss[scratch_index], /* vector for substring information */ SnortConfig::get_conf()->pcre_ovector_size); /* number of elements in the vector */ if (result >= 0) @@ -399,7 +399,7 @@ static bool pcre_search( * and a single int for scratch space. */ - found_offset = ss->pcre_ovector[1]; + found_offset = ((int*)ss[scratch_index])[1]; } else if (result == PCRE_ERROR_NOMATCH) { @@ -598,32 +598,6 @@ bool PcreOption::retry(Cursor&) return true; // continue } -//------------------------------------------------------------------------- -// public methods -//------------------------------------------------------------------------- - -void pcre_setup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - SnortState* ss = sc->state + i; - ss->pcre_ovector = (int*)snort_calloc(s_ovector_max, sizeof(int)); - } -} - -void pcre_cleanup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - SnortState* ss = sc->state + i; - - if ( ss->pcre_ovector ) - snort_free(ss->pcre_ovector); - - ss->pcre_ovector = nullptr; - } -} - //------------------------------------------------------------------------- // module //------------------------------------------------------------------------- @@ -643,7 +617,11 @@ class PcreModule : public Module { public: PcreModule() : Module(s_name, s_help, s_params) - { data = nullptr; } + { + data = nullptr; + scratch_index = SnortConfig::request_scratch( + PcreModule::scratch_setup, PcreModule::scratch_cleanup); + } ~PcreModule() override { delete data; } @@ -661,6 +639,8 @@ public: private: PcreData* data; + static void scratch_setup(SnortConfig* sc); + static void scratch_cleanup(SnortConfig* sc); }; PcreData* PcreModule::get_data() @@ -687,6 +667,28 @@ bool PcreModule::set(const char*, Value& v, SnortConfig*) return true; } +void PcreModule::scratch_setup(SnortConfig* sc) +{ + for ( unsigned i = 0; i < sc->num_slots; ++i ) + { + std::vector& ss = sc->state[i]; + ss[scratch_index] = snort_calloc(s_ovector_max, sizeof(int)); + } +} + +void PcreModule::scratch_cleanup(SnortConfig* sc) +{ + for ( unsigned i = 0; i < sc->num_slots; ++i ) + { + std::vector& ss = sc->state[i]; + + if ( ss[scratch_index] ) + snort_free(ss[scratch_index]); + + ss[scratch_index] = nullptr; + } +} + //------------------------------------------------------------------------- // api methods //------------------------------------------------------------------------- @@ -755,5 +757,12 @@ static const IpsApi pcre_api = pcre_verify }; -const BaseApi* ips_pcre = &pcre_api.base; - +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else +const BaseApi* ips_pcre[] = +#endif +{ + &pcre_api.base, + nullptr +}; diff --git a/src/ips_options/ips_pcre.h b/src/ips_options/ips_pcre.h deleted file mode 100644 index f6faa73e4..000000000 --- a/src/ips_options/ips_pcre.h +++ /dev/null @@ -1,32 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2018 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifndef IPS_PCRE_H -#define IPS_PCRE_H - -namespace snort -{ -struct SnortConfig; -} - -void pcre_setup(snort::SnortConfig*); -void pcre_cleanup(snort::SnortConfig*); - -#endif - diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index f165fec6d..2066a63d5 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -23,8 +23,6 @@ #include "config.h" #endif -#include "ips_regex.h" - #include #include @@ -74,6 +72,7 @@ struct RegexConfig // rules. static hs_scratch_t* s_scratch = nullptr; +static unsigned scratch_index; static THREAD_LOCAL unsigned s_to = 0; static THREAD_LOCAL ProfileStats regex_perf_stats; @@ -177,14 +176,13 @@ IpsOption::EvalStatus RegexOption::eval(Cursor& c, Packet*) if ( pos > c.size() ) return NO_MATCH; - SnortState* ss = SnortConfig::get_conf()->state + get_instance_id(); - assert(ss->regex_scratch); + hs_scratch_t *ss = (hs_scratch_t *) SnortConfig::get_conf()->state[get_instance_id()][scratch_index]; s_to = 0; hs_error_t stat = hs_scan( config.db, (const char*)c.buffer()+pos, c.size()-pos, 0, - (hs_scratch_t*)ss->regex_scratch, hs_match, nullptr); + ss, hs_match, nullptr); if ( s_to and stat == HS_SCAN_TERMINATED ) { @@ -230,7 +228,11 @@ static const Parameter s_params[] = class RegexModule : public Module { public: - RegexModule() : Module(s_name, s_help, s_params) { } + RegexModule() : Module(s_name, s_help, s_params) + { + scratch_index = SnortConfig::request_scratch( + RegexModule::scratch_setup, RegexModule::scratch_cleanup); + } ~RegexModule() override; bool begin(const char*, int, SnortConfig*) override; @@ -251,6 +253,8 @@ public: private: RegexConfig config; + static void scratch_setup(SnortConfig* sc); + static void scratch_cleanup(SnortConfig* sc); }; RegexModule::~RegexModule() @@ -321,33 +325,29 @@ bool RegexModule::end(const char*, int, SnortConfig*) return true; } -//------------------------------------------------------------------------- -// public methods -//------------------------------------------------------------------------- - -void regex_setup(SnortConfig* sc) +void RegexModule::scratch_setup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index]; if ( s_scratch ) - hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss->regex_scratch); + hs_clone_scratch(s_scratch, ss); else - ss->regex_scratch = nullptr; + ss = nullptr; } } -void regex_cleanup(SnortConfig* sc) +void RegexModule::scratch_cleanup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + hs_scratch_t* ss = (hs_scratch_t*) sc->state[i][scratch_index]; - if ( ss->regex_scratch ) + if ( ss ) { - hs_free_scratch((hs_scratch_t*)ss->regex_scratch); - ss->regex_scratch = nullptr; + hs_free_scratch(ss); + ss = nullptr; } } } @@ -406,5 +406,12 @@ static const IpsApi regex_api = nullptr }; -const BaseApi* ips_regex = ®ex_api.base; - +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else +const BaseApi* ips_regex[] = +#endif +{ + ®ex_api.base, + nullptr +}; diff --git a/src/ips_options/ips_regex.h b/src/ips_options/ips_regex.h deleted file mode 100644 index 771404d8d..000000000 --- a/src/ips_options/ips_regex.h +++ /dev/null @@ -1,31 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2015-2018 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifndef IPS_REGEX_H -#define IPS_REGEX_H - -namespace snort -{ -struct SnortConfig; -} - -void regex_setup(snort::SnortConfig*); -void regex_cleanup(snort::SnortConfig*); - -#endif - diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index fd9ac7a10..4f4346b2e 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -22,8 +22,6 @@ #include "config.h" #endif -#include "ips_sd_pattern.h" - #include #include @@ -59,6 +57,7 @@ using namespace snort; // "regex" and "sd_pattern" keywords. // FIXIT-L See ips_regex.cc for more information. static hs_scratch_t* s_scratch = nullptr; +static unsigned scratch_index; struct SdStats { @@ -253,13 +252,13 @@ unsigned SdPatternOption::SdSearch(Cursor& c, Packet* p) const uint8_t* buf = c.start(); unsigned int buflen = c.length(); - SnortState* ss = SnortConfig::get_conf()->state + get_instance_id(); - assert(ss->sdpattern_scratch); + std::vector ss = SnortConfig::get_conf()->state[get_instance_id()]; + assert(ss[scratch_index]); hsContext ctx(config, p, start, buf, buflen); hs_error_t stat = hs_scan(config.db, (const char*)buf, buflen, 0, - (hs_scratch_t*)ss->sdpattern_scratch, hs_match, (void*)&ctx); + (hs_scratch_t*)ss[scratch_index], hs_match, (void*)&ctx); if ( stat == HS_SCAN_TERMINATED ) ++s_stats.terminated; @@ -303,7 +302,11 @@ static const Parameter s_params[] = class SdPatternModule : public Module { public: - SdPatternModule() : Module(s_name, s_help, s_params) { } + SdPatternModule() : Module(s_name, s_help, s_params) + { + scratch_index = SnortConfig::request_scratch( + SdPatternModule::scratch_setup, SdPatternModule::scratch_cleanup); + } bool begin(const char*, int, SnortConfig*) override; bool set(const char*, Value& v, SnortConfig*) override; @@ -326,6 +329,9 @@ public: private: SdPatternConfig config; + + static void scratch_setup(SnortConfig* sc); + static void scratch_cleanup(SnortConfig* sc); }; bool SdPatternModule::begin(const char*, int, SnortConfig*) @@ -397,29 +403,29 @@ bool SdPatternModule::end(const char*, int, SnortConfig*) // public methods //------------------------------------------------------------------------- -void sdpattern_setup(SnortConfig* sc) +void SdPatternModule::scratch_setup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + std::vector& ss = sc->state[i]; if ( s_scratch ) - hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss->sdpattern_scratch); + hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss[scratch_index]); else - ss->sdpattern_scratch = nullptr; + ss[scratch_index] = nullptr; } } -void sdpattern_cleanup(SnortConfig* sc) +void SdPatternModule::scratch_cleanup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + std::vector& ss = sc->state[i]; - if ( ss->sdpattern_scratch ) + if ( ss[scratch_index] ) { - hs_free_scratch((hs_scratch_t*)ss->sdpattern_scratch); - ss->sdpattern_scratch = nullptr; + hs_free_scratch((hs_scratch_t*)ss[scratch_index]); + ss[scratch_index] = nullptr; } } } diff --git a/src/ips_options/ips_sd_pattern.h b/src/ips_options/ips_sd_pattern.h deleted file mode 100644 index f6c6f27d4..000000000 --- a/src/ips_options/ips_sd_pattern.h +++ /dev/null @@ -1,30 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2015-2018 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifndef IPS_SD_PATTERN_H -#define IPS_SD_PATTERN_H - -namespace snort -{ -struct SnortConfig; -} - -void sdpattern_setup(snort::SnortConfig*); -void sdpattern_cleanup(snort::SnortConfig*); - -#endif diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc index 78dc7e142..87c3a70ea 100644 --- a/src/ips_options/test/ips_regex_test.cc +++ b/src/ips_options/test/ips_regex_test.cc @@ -22,8 +22,6 @@ #include "config.h" #endif -#include "ips_options/ips_regex.h" - #include "framework/base_api.h" #include "framework/counts.h" #include "framework/cursor.h" @@ -55,17 +53,28 @@ namespace snort SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; -static SnortState s_state; +static std::vector s_state; + +ScScratchFunc scratch_setup; +ScScratchFunc scratch_cleanup; SnortConfig::SnortConfig(const SnortConfig* const) { state = &s_state; - memset(state, 0, sizeof(*state)); num_slots = 1; } SnortConfig::~SnortConfig() = default; +int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +{ + scratch_setup = setup; + scratch_cleanup = cleanup; + s_state.resize(1); + + return 0; +} + SnortConfig* SnortConfig::get_conf() { return snort_conf; } @@ -269,13 +278,13 @@ TEST_GROUP(ips_regex_option) // FIXIT-L cpputest hangs or crashes in the leak detector MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); opt = get_option(" foo "); - regex_setup(snort_conf); + scratch_setup(snort_conf); } void teardown() override { IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt); - regex_cleanup(snort_conf); + scratch_cleanup(snort_conf); api->pterm(snort_conf); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } @@ -343,13 +352,13 @@ TEST_GROUP(ips_regex_option_relative) // FIXIT-L cpputest hangs or crashes in the leak detector MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); opt = get_option("\\bfoo", true); - regex_setup(snort_conf); + scratch_setup(snort_conf); } void teardown() override { IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt); - regex_cleanup(snort_conf); + scratch_cleanup(snort_conf); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 732b6ab95..5bea3ea21 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -37,7 +37,6 @@ #include "filters/sfthreshold.h" #include "hash/xhash.h" #include "helpers/process.h" -#include "ips_options/ips_pcre.h" #include "latency/latency_config.h" #include "log/messages.h" #include "managers/event_manager.h" @@ -60,12 +59,6 @@ #include "utils/util.h" #include "utils/util_cstring.h" -#ifdef HAVE_HYPERSCAN -#include "ips_options/ips_regex.h" -#include "ips_options/ips_sd_pattern.h" -#include "search_engines/hyperscan.h" -#endif - #include "snort.h" #include "thread_config.h" @@ -88,6 +81,7 @@ using namespace snort; THREAD_LOCAL SnortConfig* snort_conf = nullptr; uint32_t SnortConfig::warning_flags = 0; +static std::vector > scratch_handlers; //------------------------------------------------------------------------- // private implementation @@ -197,7 +191,7 @@ void SnortConfig::init(const SnortConfig* const other_conf, ProtocolReference* p InspectorManager::new_config(this); num_slots = ThreadConfig::get_instance_max(); - state = (SnortState*)snort_calloc(num_slots, sizeof(SnortState)); + state = new std::vector[num_slots]; profiler = new ProfilerConfig; latency = new LatencyConfig(); @@ -251,12 +245,16 @@ SnortConfig::~SnortConfig() FreeClassifications(classifications); FreeReferences(references); -#ifdef HAVE_HYPERSCAN - hyperscan_cleanup(this); - sdpattern_cleanup(this); - regex_cleanup(this); -#endif - pcre_cleanup(this); + // Only call scratch cleanup if we actually called scratch setup + if ( state[0].size() > 0 ) + { + for ( unsigned i = scratch_handlers.size(); i > 0; i-- ) + { + if ( scratch_handlers[i - 1].second ) + scratch_handlers[i - 1].second(this); + } + // FIXME-T: Do we need to shrink_to_fit() state->scratch at this point? + } FreeRuleLists(this); OtnLookupFree(otn_map); @@ -292,7 +290,7 @@ SnortConfig::~SnortConfig() delete policy_map; InspectorManager::delete_config(this); - snort_free(state); + delete[] state; delete thread_config; if (gtp_ports) @@ -335,15 +333,20 @@ void SnortConfig::setup() void SnortConfig::post_setup() { - // FIXIT-L register setup and cleanup to eliminate explicit calls and - // allow pcre, regex, and hyperscan to be built dynamically. Hyperscan setup - // moved to post_setup to ensure all the prep_patterns are called before it. - pcre_setup(this); -#ifdef HAVE_HYPERSCAN - regex_setup(this); - sdpattern_setup(this); - hyperscan_setup(this); -#endif + unsigned i; + unsigned int handler_count = scratch_handlers.size(); + + // Ensure we have allocated the scratch space vector for each thread + for ( i = 0; i < num_slots; ++i ) + { + state[i].resize(handler_count); + } + + for ( i = 0; i < handler_count; ++i ) + { + if ( scratch_handlers[i].first ) + scratch_handlers[i].first(this); + } } void SnortConfig::clone(const SnortConfig* const conf) @@ -478,9 +481,9 @@ void SnortConfig::merge(SnortConfig* cmd_line) // FIXIT-M should cmd_line use the same var list / table? var_list = nullptr; - snort_free(state); + delete[] state; num_slots = ThreadConfig::get_instance_max(); - state = (SnortState*)snort_calloc(num_slots, sizeof(SnortState)); + state = new std::vector[num_slots]; } bool SnortConfig::verify() @@ -1015,6 +1018,15 @@ bool SnortConfig::tunnel_bypass_enabled(uint8_t proto) return (!((get_conf()->tunnel_mask & proto) or SFDAQ::get_tunnel_bypass(proto))); } +SO_PUBLIC int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +{ + scratch_handlers.push_back(std::make_pair(setup, cleanup)); + + // We return an index that the caller uses to reference their per thread + // scratch space + return scratch_handlers.size() - 1; +} + SO_PUBLIC SnortConfig* SnortConfig::get_conf() { return snort_conf; } diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 164ce91b2..6546d9e83 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -144,21 +144,8 @@ namespace snort { struct ProfilerConfig; -// SnortState members are updated during runtime. an array in SnortConfig is -// used instead of thread_locals because these must get changed on reload -// FIXIT-L register this data to avoid explicit dependency -struct SnortState -{ - int* pcre_ovector; - - // regex hyperscan and sdpattern are conditionally built but these are - // unconditional to avoid compatibility issues with plugins. if these are - // conditional then API_OPTIONS must be updated. - // note: fwd decls don't work here. - void* regex_scratch; - void* hyperscan_scratch; - void* sdpattern_scratch; -}; +struct SnortConfig; +typedef void (* ScScratchFunc)(SnortConfig* sc); struct SnortConfig { @@ -371,7 +358,7 @@ public: MemoryConfig* memory = nullptr; //------------------------------------------------------ - SnortState* state = nullptr; + std::vector* state = nullptr; unsigned num_slots = 0; ThreadConfig* thread_config; @@ -652,6 +639,10 @@ public: bool track_on_syn() const { return (run_flags & RUN_FLAG__TRACK_ON_SYN) != 0; } + // This requests an entry in the scratch space vector and calls setup / + // cleanup as appropriate + SO_PUBLIC static int request_scratch(ScScratchFunc setup, ScScratchFunc cleanup); + // Use this to access current thread's conf from other units static void set_conf(SnortConfig*); diff --git a/src/search_engines/CMakeLists.txt b/src/search_engines/CMakeLists.txt index 69119d588..05b9ede43 100644 --- a/src/search_engines/CMakeLists.txt +++ b/src/search_engines/CMakeLists.txt @@ -29,7 +29,6 @@ set (BNFA_SOURCES if ( HAVE_HYPERSCAN ) set(HYPER_SOURCES hyperscan.cc - hyperscan.h ) endif () @@ -39,13 +38,13 @@ set (SEARCH_ENGINE_SOURCES search_engines.h search_tool.cc ${BNFA_SOURCES} - ${HYPER_SOURCES} ) if ( STATIC_SEARCH_ENGINES ) add_library(search_engines OBJECT ${ACSMX_SOURCES} ${ACSMX2_SOURCES} + ${HYPER_SOURCES} ${INTEL_SOURCES} ${SEARCH_ENGINE_SOURCES} ${SEARCH_ENGINE_INCLUDES} @@ -59,6 +58,9 @@ else ( STATIC_SEARCH_ENGINES) add_dynamic_module(acsmx search_engines ${ACSMX_SOURCES}) add_dynamic_module(acsmx2 search_engines ${ACSMX2_SOURCES}) +if ( HAVE_HYPERSCAN ) + add_dynamic_module(hyperscan search_engines ${HYPER_SOURCES}) +endif () endif (STATIC_SEARCH_ENGINES) diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc index acd522086..53f4fc1fd 100644 --- a/src/search_engines/hyperscan.cc +++ b/src/search_engines/hyperscan.cc @@ -22,8 +22,6 @@ #include "config.h" #endif -#include "hyperscan.h" - #include #include @@ -100,6 +98,8 @@ typedef std::vector PatternVector; // a prototype that is large enough for all uses. static hs_scratch_t* s_scratch = nullptr; +static unsigned int scratch_index; +static bool scratch_registered = false; //------------------------------------------------------------------------- // mpse @@ -280,31 +280,27 @@ int HyperscanMpse::_search( match_cb = mf; match_ctx = pv; - SnortState* ss = SnortConfig::get_conf()->state + get_instance_id(); + hs_scratch_t *ss = (hs_scratch_t *) SnortConfig::get_conf()->state[get_instance_id()][scratch_index]; // scratch is null for the degenerate case w/o patterns - assert(!hs_db or ss->hyperscan_scratch); + assert(!hs_db or ss); - hs_scan(hs_db, (const char*)buf, n, 0, (hs_scratch_t*)ss->hyperscan_scratch, + hs_scan(hs_db, (const char*)buf, n, 0, ss, HyperscanMpse::match, this); return nfound; } -//------------------------------------------------------------------------- -// public methods -//------------------------------------------------------------------------- - -void hyperscan_setup(SnortConfig* sc) +static void scratch_setup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index]; if ( s_scratch ) - hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss->hyperscan_scratch); + hs_clone_scratch(s_scratch, ss); else - ss->hyperscan_scratch = nullptr; + ss = nullptr; } if ( s_scratch ) { @@ -313,16 +309,16 @@ void hyperscan_setup(SnortConfig* sc) } } -void hyperscan_cleanup(SnortConfig* sc) +static void scratch_cleanup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - SnortState* ss = sc->state + i; + hs_scratch_t* ss = (hs_scratch_t*) sc->state[i][scratch_index]; - if ( ss->hyperscan_scratch ) + if ( ss ) { - hs_free_scratch((hs_scratch_t*)ss->hyperscan_scratch); - ss->hyperscan_scratch = nullptr; + hs_free_scratch(ss); + ss = nullptr; } } } @@ -334,6 +330,11 @@ void hyperscan_cleanup(SnortConfig* sc) static Mpse* hs_ctor( SnortConfig* sc, class Module*, const MpseAgent* a) { + if ( !scratch_registered ) + { + scratch_index = SnortConfig::request_scratch(scratch_setup, scratch_cleanup); + scratch_registered = true; + } return new HyperscanMpse(sc, a); } @@ -379,11 +380,11 @@ static const MpseApi hs_api = hs_print, }; -//#ifdef BUILDING_SO -//SO_PUBLIC const BaseApi* snort_plugins[] = -//#else +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else const BaseApi* se_hyperscan[] = -//#endif +#endif { &hs_api.base, nullptr diff --git a/src/search_engines/hyperscan.h b/src/search_engines/hyperscan.h deleted file mode 100644 index cdf624bb2..000000000 --- a/src/search_engines/hyperscan.h +++ /dev/null @@ -1,32 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2015-2018 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// hyperscan.h author Russ Combs - -#ifndef HYPERSCAN_H -#define HYPERSCAN_H - -namespace snort -{ -struct SnortConfig; -} - -void hyperscan_setup(snort::SnortConfig*); -void hyperscan_cleanup(snort::SnortConfig*); - -#endif diff --git a/src/search_engines/search_engines.cc b/src/search_engines/search_engines.cc index af06a30ab..54e693672 100644 --- a/src/search_engines/search_engines.cc +++ b/src/search_engines/search_engines.cc @@ -28,26 +28,24 @@ using namespace snort; extern const BaseApi* se_ac_bnfa[]; -#ifdef HAVE_HYPERSCAN -extern const BaseApi* se_hyperscan[]; -#endif - #ifdef STATIC_SEARCH_ENGINES extern const BaseApi* se_ac_std[]; extern const BaseApi* se_acsmx2[]; +#ifdef HAVE_HYPERSCAN +extern const BaseApi* se_hyperscan[]; +#endif #endif void load_search_engines() { PluginManager::load_plugins(se_ac_bnfa); -#ifdef HAVE_HYPERSCAN - PluginManager::load_plugins(se_hyperscan); -#endif - #ifdef STATIC_SEARCH_ENGINES PluginManager::load_plugins(se_ac_std); PluginManager::load_plugins(se_acsmx2); +#ifdef HAVE_HYPERSCAN + PluginManager::load_plugins(se_hyperscan); +#endif #endif } diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index 486996a17..b32c503ee 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -22,8 +22,6 @@ #include "config.h" #endif -#include "search_engines/hyperscan.h" - #include #include "framework/base_api.h" @@ -60,17 +58,28 @@ int Mpse::search_all( SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; -static SnortState s_state; +static std::vector s_state; + +ScScratchFunc scratch_setup; +ScScratchFunc scratch_cleanup; SnortConfig::SnortConfig(const SnortConfig* const) { state = &s_state; - memset(state, 0, sizeof(*state)); num_slots = 1; } SnortConfig::~SnortConfig() = default; +int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +{ + scratch_setup = setup; + scratch_cleanup = cleanup; + s_state.resize(1); + + return 0; +} + SnortConfig* SnortConfig::get_conf() { return snort_conf; } @@ -183,7 +192,7 @@ TEST_GROUP(mpse_hs_match) void teardown() override { mpse_api->dtor(hs); - hyperscan_cleanup(snort_conf); + scratch_cleanup(snort_conf); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; @@ -207,7 +216,7 @@ TEST(mpse_hs_match, single) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -222,7 +231,7 @@ TEST(mpse_hs_match, nocase) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -238,7 +247,7 @@ TEST(mpse_hs_match, other) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -256,7 +265,7 @@ TEST(mpse_hs_match, multi) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 3); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 3); @@ -273,7 +282,7 @@ TEST(mpse_hs_match, regex) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 0); @@ -290,7 +299,7 @@ TEST(mpse_hs_match, pcre) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)":definition(", 12, match, nullptr, &state) == 0); @@ -330,7 +339,7 @@ TEST_GROUP(mpse_hs_multi) { mpse_api->dtor(hs1); mpse_api->dtor(hs2); - hyperscan_cleanup(snort_conf); + scratch_cleanup(snort_conf); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; @@ -348,7 +357,7 @@ TEST(mpse_hs_multi, single) CHECK(hs1->get_pattern_count() == 1); CHECK(hs2->get_pattern_count() == 1); - hyperscan_setup(snort_conf); + scratch_setup(snort_conf); int state = 0; CHECK(hs1->search((uint8_t*)"fubar", 5, match, nullptr, &state) == 1 ); diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc index 9adca659d..0bf7084b3 100644 --- a/src/search_engines/test/search_tool_test.cc +++ b/src/search_engines/test/search_tool_test.cc @@ -50,12 +50,11 @@ SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; -static SnortState s_state; +static std::vector s_state; SnortConfig::SnortConfig(const SnortConfig* const) { state = &s_state; - memset(state, 0, sizeof(*state)); num_slots = 1; fast_pattern_config = nullptr; }