From: Russ Combs (rucombs) Date: Fri, 24 Jan 2020 17:33:34 +0000 (+0000) Subject: Merge pull request #1890 in SNORT/snort3 from ~DAVMCPHE/snort3:hyper_vs_pcre to master X-Git-Tag: 3.0.0-268~43 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b398a1cf0f41fc23585857178f4093cb87cc9bff;p=thirdparty%2Fsnort3.git Merge pull request #1890 in SNORT/snort3 from ~DAVMCPHE/snort3:hyper_vs_pcre to master Squashed commit of the following: commit 3b003e8e351bacc4eb161515615dd2a6b3736933 Author: davis mcpherson Date: Thu Nov 14 16:55:24 2019 -0500 ips_pcre: compile/evaluate pcre rule option regular expressions with the hyperscan regex engine when possible ips_pcre: support the O & R modifiers when converting pcre to regex detection: add config option to enable conversion of pcre expressions to use the regex engine --- diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index 83b9abc7d..ca4c0143a 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -30,10 +30,13 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" +#include "framework/parameter.h" #include "hash/hashfcn.h" #include "helpers/scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" +#include "managers/ips_manager.h" +#include "managers/module_manager.h" #include "profiler/profiler.h" #include "utils/util.h" @@ -59,6 +62,7 @@ using namespace snort; #define SNORT_OVERRIDE_MATCH_LIMIT 0x00080 // Override default limits on match & match recursion #define s_name "pcre" +#define mod_regex_name "regex" struct PcreData { @@ -571,6 +575,7 @@ IpsOption::EvalStatus PcreOption::eval(Cursor& c, Packet*) } return MATCH; } + return NO_MATCH; } @@ -606,6 +611,25 @@ static const Parameter s_params[] = { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; +struct PcreStats +{ + PegCount pcre_rules; + PegCount pcre_to_hyper; + PegCount pcre_native; + PegCount pcre_negated; +}; + +const PegInfo pcre_pegs[] = +{ + { CountType::SUM, "pcre_rules", "total rules processed with pcre option" }, + { CountType::SUM, "pcre_to_hyper", "total pcre rules by hyperscan engine" }, + { CountType::SUM, "pcre_native", "total pcre rules compiled by pcre engine" }, + { CountType::SUM, "pcre_negated", "total pcre rules using negation syntax" }, + { CountType::END, nullptr, nullptr } +}; + +PcreStats pcre_stats; + #define s_help \ "rule option for matching payload data with pcre" @@ -627,17 +651,28 @@ public: bool begin(const char*, int, SnortConfig*) override; bool set(const char*, Value&, SnortConfig*) override; + bool end(const char*, int, SnortConfig*) override; ProfileStats* get_profile() const override { return &pcrePerfStats; } + const PegInfo* get_pegs() const override; + PegCount* get_counts() const override; + PcreData* get_data(); Usage get_usage() const override { return DETECT; } + void get_mod_regex_instance(const char* name, int v, SnortConfig* sc); + Module* get_mod_regex() const + { return mod_regex; } + private: PcreData* data; + Module* mod_regex = nullptr; + std::string re; + static bool scratch_setup(SnortConfig* sc); static void scratch_cleanup(SnortConfig* sc); }; @@ -649,23 +684,59 @@ PcreData* PcreModule::get_data() return tmp; } -bool PcreModule::begin(const char*, int, SnortConfig*) +const PegInfo* PcreModule::get_pegs() const +{ return pcre_pegs; } + +PegCount* PcreModule::get_counts() const +{ return (PegCount*)&pcre_stats; } + +void PcreModule::get_mod_regex_instance(const char* name, int v, SnortConfig* sc) +{ + if ( sc->pcre_to_regex ) + { + if ( !mod_regex ) + mod_regex = ModuleManager::get_module(mod_regex_name); + + if( mod_regex ) + mod_regex = mod_regex->begin(name, v, sc) ? mod_regex : nullptr; + } +} + +bool PcreModule::begin(const char* name, int v, SnortConfig* sc) { - data = (PcreData*)snort_calloc(sizeof(*data)); + get_mod_regex_instance(name, v, sc); return true; } -bool PcreModule::set(const char*, Value& v, SnortConfig* sc) +bool PcreModule::set(const char* name, Value& v, SnortConfig* sc) { if ( v.is("~re") ) - pcre_parse(sc, v.get_string(), data); + { + re = v.get_string(); + if( mod_regex ) + mod_regex = mod_regex->set(name, v, sc) ? mod_regex : nullptr; + } else return false; return true; } +bool PcreModule::end(const char* name, int v, SnortConfig* sc) +{ + if( mod_regex ) + mod_regex = mod_regex->end(name, v, sc) ? mod_regex : nullptr; + + if ( !mod_regex ) + { + data = (PcreData*)snort_calloc(sizeof(*data)); + pcre_parse(sc, re.c_str(), data); + } + + return true; +} + bool PcreModule::scratch_setup(SnortConfig* sc) { if ( s_ovector_max < 0 ) @@ -703,26 +774,33 @@ void PcreModule::scratch_cleanup(SnortConfig* sc) //------------------------------------------------------------------------- static Module* mod_ctor() -{ - return new PcreModule; -} +{ return new PcreModule; } static void mod_dtor(Module* m) -{ - delete m; -} +{ delete m; } -static IpsOption* pcre_ctor(Module* p, OptTreeNode*) +static IpsOption* pcre_ctor(Module* p, OptTreeNode* otn) { + pcre_stats.pcre_rules++; PcreModule* m = (PcreModule*)p; - PcreData* d = m->get_data(); - return new PcreOption(d); + + Module* mod_regex = m->get_mod_regex(); + if ( mod_regex ) + { + pcre_stats.pcre_to_hyper++; + const IpsApi* opt_api = IpsManager::get_option_api(mod_regex_name); + return opt_api->ctor(mod_regex, otn); + } + else + { + pcre_stats.pcre_native++; + PcreData* d = m->get_data(); + return new PcreOption(d); + } } static void pcre_dtor(IpsOption* p) -{ - delete p; -} +{ delete p; } static const IpsApi pcre_api = { diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index a86f106cf..bdb37efe2 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -50,6 +50,7 @@ struct RegexConfig hs_database_t* db; std::string re; PatternMatchData pmd; + bool pcre_conversion; RegexConfig() { reset(); } @@ -59,6 +60,7 @@ struct RegexConfig memset(&pmd, 0, sizeof(pmd)); re.clear(); db = nullptr; + pcre_conversion = false; } }; @@ -128,18 +130,18 @@ uint32_t RegexOption::hash() const // see ContentOption::operator==() bool RegexOption::operator==(const IpsOption& ips) const { -#if 0 if ( !IpsOption::operator==(ips) ) return false; - RegexOption& rhs = (RegexOption&)ips; + const RegexOption& rhs = (const RegexOption&)ips; - if ( config.re == rhs.config.re and - config.pmd.flags == rhs.config.pmd.flags and - config.pmd.mpse_flags == rhs.config.pmd.mpse_flags ) - return true; -#endif - return this == &ips; + if ( config.pcre_conversion && rhs.config.pcre_conversion ) + if ( config.re == rhs.config.re and + config.pmd.flags == rhs.config.pmd.flags and + config.pmd.mpse_flags == rhs.config.pmd.mpse_flags ) + return true; + + return false; } struct ScanContext @@ -187,9 +189,7 @@ IpsOption::EvalStatus RegexOption::eval(Cursor& c, Packet*) } bool RegexOption::retry(Cursor&) -{ - return !is_relative(); -} +{ return !is_relative(); } //------------------------------------------------------------------------- // module @@ -244,6 +244,7 @@ public: private: RegexConfig config; + bool convert_pcre_to_regex_form(); }; RegexModule::~RegexModule() @@ -254,26 +255,96 @@ RegexModule::~RegexModule() delete scratcher; } -bool RegexModule::begin(const char*, int, SnortConfig*) +bool RegexModule::begin(const char* name, int, SnortConfig*) { config.reset(); config.pmd.flags |= PatternMatchData::NO_FP; config.pmd.mpse_flags |= HS_FLAG_SINGLEMATCH; + + // if regex is in pcre syntax set conversion mode + if ( strcmp(name, "pcre") == 0 ) + config.pcre_conversion = true; + return true; } +// The regex string received from the ips_pcre plugin must be scrubbed to remove +// two characters from the front; an extra '"' and the '/' and also the same +// two characters from the end of the string as well as any pcre modifier flags +// included in the expression. The modifier flags are checked to set the +// corresponding hyperscan regex engine flags. +// Hyperscan regex also does not support negated pcre expression so negated expression +// are not converted and will be compiled by the pcre engine. +bool RegexModule::convert_pcre_to_regex_form() +{ + size_t pos = config.re.find_first_of("\"!"); + if (config.re[pos] == '!') + return false; + + config.re.erase(0,2); + std::size_t re_end = config.re.rfind("/"); + if ( re_end != std::string::npos ) + { + std::size_t mod_len = (config.re.length() - 2) - re_end; + std::string modifiers = config.re.substr(re_end + 1, mod_len); + std::size_t erase_len = config.re.length() - re_end; + config.re.erase(re_end, erase_len); + + for( char& c : modifiers ) + { + switch (c) + { + case 'i': + config.pmd.mpse_flags |= HS_FLAG_CASELESS; + config.pmd.set_no_case(); + break; + + case 'm': + config.pmd.mpse_flags |= HS_FLAG_MULTILINE; + break; + + case 's': + config.pmd.mpse_flags |= HS_FLAG_DOTALL; + break; + + case 'O': + break; + + case 'R': + config.pmd.set_relative(); + break; + + default: + return false; + break; + } + } + + return true; + } + + return false; +} + bool RegexModule::set(const char*, Value& v, SnortConfig*) { + bool valid_opt = true; + if ( v.is("~re") ) { config.re = v.get_string(); - // remove quotes - config.re.erase(0, 1); - config.re.erase(config.re.length()-1, 1); + + if ( config.pcre_conversion ) + valid_opt = convert_pcre_to_regex_form(); + else + { + // remove quotes + config.re.erase(0, 1); + config.re.erase(config.re.length() - 1, 1); + } } else if ( v.is("dotall") ) config.pmd.mpse_flags |= HS_FLAG_DOTALL; - else if ( v.is("fast_pattern") ) { config.pmd.flags &= ~PatternMatchData::NO_FP; @@ -281,7 +352,6 @@ bool RegexModule::set(const char*, Value& v, SnortConfig*) } else if ( v.is("multiline") ) config.pmd.mpse_flags |= HS_FLAG_MULTILINE; - else if ( v.is("nocase") ) { config.pmd.mpse_flags |= HS_FLAG_CASELESS; @@ -289,11 +359,10 @@ bool RegexModule::set(const char*, Value& v, SnortConfig*) } else if ( v.is("relative") ) config.pmd.set_relative(); - else - return false; + valid_opt = false; - return true; + return valid_opt; } bool RegexModule::end(const char*, int, SnortConfig*) @@ -309,7 +378,8 @@ bool RegexModule::end(const char*, int, SnortConfig*) if ( hs_compile(config.re.c_str(), config.pmd.mpse_flags, HS_MODE_BLOCK, nullptr, &config.db, &err) or !config.db ) { - ParseError("can't compile regex '%s'", config.re.c_str()); + if ( !config.pcre_conversion ) + ParseError("can't compile regex '%s'", config.re.c_str()); hs_free_compile_error(err); return false; } diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc index 37fa965d2..1fcaef00e 100644 --- a/src/ips_options/test/ips_regex_test.cc +++ b/src/ips_options/test/ips_regex_test.cc @@ -292,7 +292,6 @@ TEST(ips_regex_option, hash) { IpsOption* opt2 = get_option(mod, "bar"); CHECK(opt2); - CHECK(*opt != *opt2); uint32_t h1 = opt->hash(); uint32_t h2 = opt2->hash(); @@ -308,8 +307,6 @@ TEST(ips_regex_option, opeq) { IpsOption* opt2 = get_option(mod, " foo "); CHECK(opt2); - // this is forced unequal for now - CHECK(*opt != *opt2); do_cleanup = scratcher->setup(snort_conf); diff --git a/src/main/modules.cc b/src/main/modules.cc index d4f3afd22..76cf6a161 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -104,6 +104,9 @@ static const Parameter detection_params[] = { "pcre_override", Parameter::PT_BOOL, nullptr, "true", "enable pcre match limit overrides when pattern matching (ie ignore /O)" }, + { "pcre_to_regex", Parameter::PT_BOOL, nullptr, "false", + "disable pcre pattern matching" }, + { "enable_address_anomaly_checks", Parameter::PT_BOOL, nullptr, "false", "enable check and alerting of address anomalies" }, @@ -205,6 +208,9 @@ bool DetectionModule::set(const char* fqn, Value& v, SnortConfig* sc) else if ( v.is("pcre_override") ) sc->pcre_override = v.get_bool(); + + else if ( v.is("pcre_to_regex") ) + sc->pcre_to_regex = v.get_bool(); else if ( v.is("enable_address_anomaly_checks") ) sc->address_anomaly_check_enabled = v.get_bool(); diff --git a/src/main/snort_config.h b/src/main/snort_config.h index a92e76b35..c8ff51dd5 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -248,6 +248,7 @@ public: bool global_rule_state = false; bool global_default_rule_state = true; + bool pcre_to_regex = false; //------------------------------------------------------ // process stuff diff --git a/src/managers/ips_manager.cc b/src/managers/ips_manager.cc index 3a2eeaacd..3be97563d 100644 --- a/src/managers/ips_manager.cc +++ b/src/managers/ips_manager.cc @@ -176,6 +176,15 @@ const char* IpsManager::get_option_keyword() return current_keyword.c_str(); } +const IpsApi* IpsManager::get_option_api(const char* keyword) +{ + Option* opt = get_opt(keyword); + if ( opt ) + return opt->api; + else + return nullptr; +} + bool IpsManager::option_begin( SnortConfig* sc, const char* key, SnortProtocolId) { diff --git a/src/managers/ips_manager.h b/src/managers/ips_manager.h index 3ba690b13..0f9e3eba7 100644 --- a/src/managers/ips_manager.h +++ b/src/managers/ips_manager.h @@ -59,25 +59,17 @@ public: static void add_plugin(const snort::IpsApi*); static void dump_plugins(); static void release_plugins(); - static void instantiate(const snort::IpsApi*, snort::Module*, snort::SnortConfig*); - - static bool get_option( - snort::SnortConfig*, struct OptTreeNode*, SnortProtocolId, - const char* keyword, char* args, snort::RuleOptType&); - static bool option_begin(snort::SnortConfig*, const char* key, SnortProtocolId); static bool option_set( snort::SnortConfig*, const char* key, const char* opt, const char* val); static bool option_end( snort::SnortConfig*, OptTreeNode*, SnortProtocolId, const char* key, snort::RuleOptType&); - static void delete_option(snort::IpsOption*); static const char* get_option_keyword(); - + SO_PUBLIC static const snort::IpsApi* get_option_api(const char* keyword); static void global_init(snort::SnortConfig*); static void global_term(snort::SnortConfig*); - static void reset_options(); static void setup_options(); static void clear_options(); diff --git a/src/stream/tcp/tcp_module.cc b/src/stream/tcp/tcp_module.cc index 976473122..808c610a3 100644 --- a/src/stream/tcp/tcp_module.cc +++ b/src/stream/tcp/tcp_module.cc @@ -232,7 +232,6 @@ StreamTcpModule::StreamTcpModule() : config = nullptr; } - const RuleMap* StreamTcpModule::get_rules() const { return stream_tcp_rules; }