From: Russ Combs (rucombs) Date: Thu, 29 Oct 2015 22:13:07 +0000 (-0400) Subject: Merge pull request #102 in SNORT/snort3 from crc/regex to master X-Git-Tag: 3.0.0-233~758 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b5f2be2052d79cd4664c06006d32b8b3ff42df5c;p=thirdparty%2Fsnort3.git Merge pull request #102 in SNORT/snort3 from crc/regex to master Squashed commit of the following: commit 9686d11d5a0c4afac1430563afcc6d352f67782e Author: russ Date: Wed Oct 28 12:46:59 2015 -0400 initial hyperscan / proof of concept add regex rule option for basic matching rule tree and fast pattern updates will follow --- diff --git a/cmake/include_libraries.cmake b/cmake/include_libraries.cmake index a53a3407d..e9ad1b0a1 100644 --- a/cmake/include_libraries.cmake +++ b/cmake/include_libraries.cmake @@ -9,6 +9,7 @@ find_package(PCRE REQUIRED) find_package(ZLIB REQUIRED) # optional libraries +find_package(LibHS QUIET) find_package(LibLZMA QUIET) find_package(OpenSSL QUIET) find_package(Asciidoc QUIET) diff --git a/cmake/sanity_checks.cmake b/cmake/sanity_checks.cmake index 27b1f3272..c11f7cee2 100644 --- a/cmake/sanity_checks.cmake +++ b/cmake/sanity_checks.cmake @@ -189,6 +189,10 @@ if (DEFINED OPENSSL_CRYPTO_LIBRARIES) check_library_exists (${OPENSSL_CRYPTO_LIBRARIES} MD5_Init "" HAVE_OPENSSL_MD5) endif() +if (DEFINED LIBHS_LIBRARIES) + check_library_exists (${LIBHS_LIBRARIES} hs_scan "" HAVE_HYPERSCAN) +endif() + if (DEFINED LIBLZMA_LIBRARIES) check_library_exists (${LIBLZMA_LIBRARIES} lzma_code "" HAVE_LZMA) endif() diff --git a/configure.ac b/configure.ac index 5b1773975..2a51968a5 100644 --- a/configure.ac +++ b/configure.ac @@ -917,6 +917,38 @@ if test "x$enable_intel_soft_cpm" = "xyes"; then AC_DEFINE(INTEL_SOFT_CPM, [1], [enable intel cpm support in build]) fi +#-------------------------------------------------------------------------- +# hyperscan (optional) +#-------------------------------------------------------------------------- + +AC_ARG_WITH(hyperscan_includes, + AC_HELP_STRING([--with-hyperscan-includes=DIR],[libhs include directory]), + [with_hyperscan_includes="$withval"],[with_hyperscan_includes="no"]) + +if test "x$with_hyperscan_includes" != "xno"; then + CPPFLAGS="${CPPFLAGS} -I${with_hyperscan_includes}" +fi + +AC_ARG_WITH(hyperscan_libraries, + AC_HELP_STRING([--with-hyperscan-libraries=DIR],[libhs library directory]), + [with_hyperscan_libraries="$withval"],[with_hyperscan_libraries="no"]) + +if test "x$with_hyperscan_libraries" != "xno"; then + LDFLAGS="${LDFLAGS} -L${with_hyperscan_libraries}" +fi + +AC_CHECK_HEADERS(hs/hs.h, HS_HEADERS="yes", HS_HEADERS="no") +AC_CHECK_LIB(hs, hs_scan, HS_LIB="yes", HS_LIB="no") + +if test "x$HS_LIB" != "xno"; then + if test "x$HS_HEADERS" != "xno"; then + AC_DEFINE([HAVE_HYPERSCAN],[1],[can build hyperscan code]) + LIBS="${LIBS} -lhs" + fi +fi + +AM_CONDITIONAL([HAVE_HYPERSCAN], [test "x$HS_HEADERS" = "xyes" -a "x$HS_LIB" = "xyes"]) + #-------------------------------------------------------------------------- # outputs #-------------------------------------------------------------------------- @@ -957,6 +989,7 @@ src/hash/Makefile \ src/helpers/Makefile \ src/lua/Makefile \ src/ips_options/Makefile \ +src/ips_options/test/Makefile \ src/log/Makefile \ src/loggers/Makefile \ src/main/Makefile \ diff --git a/src/ips_options/Makefile.am b/src/ips_options/Makefile.am index 2665a3dd3..384194253 100644 --- a/src/ips_options/Makefile.am +++ b/src/ips_options/Makefile.am @@ -40,6 +40,10 @@ ips_ttl.cc \ ips_bufferlen.cc \ ips_window.cc +if HAVE_HYPERSCAN +plugin_list += ips_regex.cc +endif + libips_options_a_SOURCES = \ ips_byte_extract.cc ips_byte_extract.h \ extract.cc extract.h \ @@ -196,6 +200,13 @@ libips_raw_data_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO libips_raw_data_la_LDFLAGS = -export-dynamic -shared libips_raw_data_la_SOURCES = ips_raw_data.cc +if HAVE_HYPERSCAN +optlib_LTLIBRARIES += libips_regex.la +libips_regex_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO +libips_regex_la_LDFLAGS = -export-dynamic -shared +libips_regex_la_SOURCES = ips_regex.cc +endif + optlib_LTLIBRARIES += libips_rem.la libips_rem_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO libips_rem_la_LDFLAGS = -export-dynamic -shared @@ -253,3 +264,8 @@ libips_window_la_SOURCES = ips_window.cc endif AM_CXXFLAGS = @AM_CXXFLAGS@ + +if BUILD_UNIT_TESTS +SUBDIRS = test +endif + diff --git a/src/ips_options/ips_options.cc b/src/ips_options/ips_options.cc index 246de7d7b..99addaa2e 100644 --- a/src/ips_options/ips_options.cc +++ b/src/ips_options/ips_options.cc @@ -73,6 +73,9 @@ extern const BaseApi* ips_md5; extern const BaseApi* ips_msg; extern const BaseApi* ips_priority; extern const BaseApi* ips_raw_data; +#ifdef HAVE_HYPERSCAN +extern const BaseApi* ips_regex; +#endif extern const BaseApi* ips_rem; extern const BaseApi* ips_rev; extern const BaseApi* ips_rpc; @@ -139,6 +142,9 @@ const BaseApi* ips_options[] = ips_msg, ips_priority, ips_raw_data, +#ifdef HAVE_HYPERSCAN + ips_regex, +#endif ips_rem, ips_rev, ips_rpc, diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc new file mode 100644 index 000000000..9ca10c0a6 --- /dev/null +++ b/src/ips_options/ips_regex.cc @@ -0,0 +1,323 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// ips_regex.cc author Russ Combs + +#include +#include + +#include +#include + +#include "framework/cursor.h" +#include "framework/ips_option.h" +#include "framework/module.h" +#include "detection/detection_defines.h" +#include "hash/sfhashfcn.h" +#include "main/thread.h" +#include "parser/parser.h" +#include "time/profiler.h" + +#define s_name "regex" + +#define s_help \ + "rule option for matching payload data with hyperscan regex" + +struct RegexConfig +{ + std::string re; + hs_database_t* db; + unsigned flags; + bool relative; + + void reset() + { + re.clear(); + db = nullptr; + flags = 0; + relative = false; + } +}; + +// we need to update scratch in the main thread as each pattern +// is processed and then clone to packet thread in tinit() +static hs_scratch_t* s_scratch = NULL; +static THREAD_LOCAL hs_scratch_t* t_scratch = NULL; +static THREAD_LOCAL unsigned s_to = 0; +static THREAD_LOCAL ProfileStats regex_perf_stats; + +//------------------------------------------------------------------------- +// option +//------------------------------------------------------------------------- + +class RegexOption : public IpsOption +{ +public: + RegexOption(RegexConfig&); + ~RegexOption(); + + uint32_t hash() const override; + bool operator==(const IpsOption&) const override; + + bool is_relative() override + { return config.relative; } + + int eval(Cursor&, Packet*) override; + +private: + RegexConfig config; +}; + +RegexOption::RegexOption(RegexConfig& c) : IpsOption(s_name, RULE_OPTION_TYPE_OTHER) +{ + config = c; + + if ( /*hs_error_t err =*/ hs_alloc_scratch(config.db, &s_scratch) ) + { + // FIXIT-H why is this failing but everything is working? + //ParseError("can't initialize regex for '%s' (%d) %p", + // config.re.c_str(), err, s_scratch); + } +} + +RegexOption::~RegexOption() +{ + if ( config.db ) + hs_free_database(config.db); +} + +uint32_t RegexOption::hash() const +{ + uint32_t a = config.flags, b = config.relative, c = 0; + mix_str(a, b, c, config.re.c_str()); + mix_str(a, b, c, get_name()); + finalize(a, b, c); + return c; +} + +bool RegexOption::operator==(const IpsOption& ips) const +{ + if ( strcmp(get_name(), ips.get_name()) ) + return false; + + RegexOption& rhs = (RegexOption&)ips; + + if ( config.re == rhs.config.re and + config.flags == rhs.config.flags and + config.relative == rhs.config.relative ) + return true; + + return false; +} + +static int hs_match( + unsigned int /*id*/, unsigned long long /*from*/, unsigned long long to, + unsigned int /*flags*/, void* /*context*/) +{ + s_to = (unsigned)to; + return 1; // stop search +} + +int RegexOption::eval(Cursor& c, Packet*) +{ + PERF_PROFILE(regex_perf_stats); + + unsigned pos = c.get_delta(); + + if ( !pos && is_relative() ) + pos = c.get_pos(); + + if ( pos > c.size() ) + return DETECTION_OPTION_NO_MATCH; + + s_to = 0; + + hs_error_t stat = hs_scan( + config.db, (char*)c.buffer()+pos, c.size()-pos, config.flags, + t_scratch, hs_match, nullptr); + + if ( s_to and stat == HS_SCAN_TERMINATED ) + { + c.set_pos(s_to); + c.set_delta(s_to); + return DETECTION_OPTION_MATCH; + } + return DETECTION_OPTION_NO_MATCH; +} + +//------------------------------------------------------------------------- +// module +//------------------------------------------------------------------------- + +static const Parameter s_params[] = +{ + { "~", Parameter::PT_STRING, nullptr, nullptr, + "hyperscan regular expression" }, + + { "nocase", Parameter::PT_IMPLIED, nullptr, nullptr, + "case insensitive match" }, + + { "dotall", Parameter::PT_IMPLIED, nullptr, nullptr, + "matching a . will not exclude newlines" }, + + { "multiline", Parameter::PT_IMPLIED, nullptr, nullptr, + "^ and $ anchors match any newlines in data" }, + + { "relative", Parameter::PT_IMPLIED, nullptr, nullptr, + "start search from end of last match instead of start of buffer" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +class RegexModule : public Module +{ +public: + RegexModule() : Module(s_name, s_help, s_params) { } + ~RegexModule(); + + bool begin(const char*, int, SnortConfig*) override; + bool end(const char*, int, SnortConfig*) override; + bool set(const char*, Value&, SnortConfig*) override; + + ProfileStats* get_profile() const override + { return ®ex_perf_stats; } + + void get_data(RegexConfig& c) + { + c = config; + config.reset(); + } + +private: + RegexConfig config; +}; + +RegexModule::~RegexModule() +{ + if ( config.db ) + hs_free_database(config.db); +} + +bool RegexModule::begin(const char*, int, SnortConfig*) +{ + config.reset(); + return true; +} + +bool RegexModule::set(const char*, Value& v, SnortConfig*) +{ + if ( v.is("~") ) + { + config.re = v.get_string(); + // remove quotes + config.re.erase(0, 1); + config.re.erase(config.re.length()-1, 1); + } + + else if ( v.is("nocase") ) + config.flags |= HS_FLAG_CASELESS; + + else if ( v.is("dotall") ) + config.flags |= HS_FLAG_DOTALL; + + else if ( v.is("multiline") ) + config.flags |= HS_FLAG_MULTILINE; + + else if ( v.is("relative") ) + config.relative = true; + + else + return false; + + return true; +} + +bool RegexModule::end(const char*, int, SnortConfig*) +{ + hs_compile_error_t* err = nullptr; + + if ( hs_compile(config.re.c_str(), config.flags, HS_MODE_BLOCK, NULL, &config.db, &err) + or !config.db ) + { + ParseError("can't compile regex '%s'", config.re.c_str()); + hs_free_compile_error(err); + return false; + } + return true; +} + +//------------------------------------------------------------------------- +// api methods +//------------------------------------------------------------------------- + +static Module* mod_ctor() +{ return new RegexModule; } + +static void mod_dtor(Module* p) +{ delete p; } + +static IpsOption* regex_ctor(Module* m, OptTreeNode*) +{ + RegexModule* mod = (RegexModule*)m; + RegexConfig c; + mod->get_data(c); + return new RegexOption(c); +} + +static void regex_dtor(IpsOption* p) +{ delete p; } + +static void regex_tinit(SnortConfig*) +{ + if ( s_scratch ) + hs_clone_scratch(s_scratch, &t_scratch); +} + +static void regex_tterm(SnortConfig*) +{ + if ( t_scratch ) + hs_free_scratch(t_scratch); +} + +static const IpsApi regex_api = +{ + { + PT_IPS_OPTION, + sizeof(IpsApi), + IPSAPI_VERSION, + 0, + API_RESERVED, + API_OPTIONS, + s_name, + s_help, + mod_ctor, + mod_dtor + }, + OPT_TYPE_DETECTION, + 0, 0, + nullptr, + nullptr, + regex_tinit, + regex_tterm, + regex_ctor, + regex_dtor, + nullptr +}; + +const BaseApi* ips_regex = ®ex_api.base; + diff --git a/src/ips_options/test/Makefile.am b/src/ips_options/test/Makefile.am new file mode 100644 index 000000000..6046610b4 --- /dev/null +++ b/src/ips_options/test/Makefile.am @@ -0,0 +1,14 @@ + +AM_DEFAULT_SOURCE_EXT = .cc + +check_PROGRAMS = \ +ips_regex_test + +TESTS = $(check_PROGRAMS) + +ips_regex_test_LDADD = \ +../../framework/ips_option.cc \ +../../framework/module.cc \ +../../framework/value.cc \ +../../sfip/sf_ip.cc + diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc new file mode 100644 index 000000000..656f8e024 --- /dev/null +++ b/src/ips_options/test/ips_regex_test.cc @@ -0,0 +1,319 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2015-2015 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// ips_regex_test.cc author Russ Combs + +#include +#include + +#include "framework/base_api.h" +#include "framework/counts.h" +#include "framework/cursor.h" +#include "framework/ips_option.h" +#include "framework/module.h" +#include "protocols/packet.h" +#include "detection/detection_defines.h" + +//------------------------------------------------------------------------- +// stubs, spies, etc. +//------------------------------------------------------------------------- + +extern const BaseApi* ips_regex; + +void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } + +void mix_str(uint32_t& a, uint32_t&, uint32_t&, const char* s, unsigned) +{ a += strlen(s); } + +Cursor::Cursor(Packet* p) +{ set("pkt_data", p->data, p->dsize); } + +static unsigned s_parse_errors = 0; + +void ParseError(const char*, ...) +{ + s_parse_errors++; +} + +//------------------------------------------------------------------------- +// helpers +//------------------------------------------------------------------------- + +static const Parameter* get_param(Module* m, const char* s) +{ + const Parameter* p = m->get_parameters(); + + while ( p and p->name ) + { + if ( !strcmp(p->name, s) ) + return p; + ++p; + } + return nullptr; +} + +static IpsOption* get_option(const char* pat, bool relative = false) +{ + Module* mod = ips_regex->mod_ctor(); + mod->begin(ips_regex->name, 0, nullptr); + + Value vs(pat); + vs.set(get_param(mod, "~")); + mod->set(ips_regex->name, vs, nullptr); + + if ( relative ) + { + Value vb(relative); + vb.set(get_param(mod, "relative")); + mod->set(ips_regex->name, vb, nullptr); + } + mod->end(ips_regex->name, 0, nullptr); + + IpsApi* api = (IpsApi*)ips_regex; + IpsOption* opt = api->ctor(mod, nullptr); + + ips_regex->mod_dtor(mod); + return opt; +} + +//------------------------------------------------------------------------- +// base tests +//------------------------------------------------------------------------- + +TEST_GROUP(ips_regex_base) +{ + void setup() + { CHECK(ips_regex); } +}; + +TEST(ips_regex_base, base) +{ + CHECK(ips_regex->type == PT_IPS_OPTION); + CHECK(ips_regex->name); + CHECK(ips_regex->help); + + CHECK(!strcmp(ips_regex->name, "regex")); + + CHECK(ips_regex->mod_ctor); + CHECK(ips_regex->mod_dtor); +} + +TEST(ips_regex_base, ips_option) +{ + const IpsApi* ips_api = (IpsApi*)ips_regex; + + CHECK(ips_api->ctor); + CHECK(ips_api->dtor); +} + +//------------------------------------------------------------------------- +// module tests +//------------------------------------------------------------------------- + +TEST_GROUP(ips_regex_module) +{ + Module* mod = nullptr; + bool end = true; + unsigned expect = 0; + + void setup() + { + s_parse_errors = 0; + mod = ips_regex->mod_ctor(); + CHECK(mod); + CHECK(mod->begin(ips_regex->name, 0, nullptr)); + } + void teardown() + { + CHECK(mod->end(ips_regex->name, 0, nullptr) == end); + LONGS_EQUAL(expect, s_parse_errors); + ips_regex->mod_dtor(mod); + } +}; + +TEST(ips_regex_module, basic) +{ + // always need a re + Value vs("foo"); + const Parameter* p = get_param(mod, "~"); + CHECK(p); + vs.set(p); + CHECK(mod->set(ips_regex->name, vs, nullptr)); + + CHECK(mod->get_profile()); +} + +TEST(ips_regex_module, config_pass) +{ + Value vs("foo"); + const Parameter* p = get_param(mod, "~"); + CHECK(p); + vs.set(p); + CHECK(mod->set(ips_regex->name, vs, nullptr)); + + Value vb(true); + p = get_param(mod, "nocase"); + CHECK(p); + vb.set(p); + CHECK(mod->set(ips_regex->name, vb, nullptr)); + + p = get_param(mod, "dotall"); + CHECK(p); + vb.set(p); + CHECK(mod->set(ips_regex->name, vb, nullptr)); + + p = get_param(mod, "multiline"); + CHECK(p); + vb.set(p); + CHECK(mod->set(ips_regex->name, vb, nullptr)); + + p = get_param(mod, "relative"); + CHECK(p); + vb.set(p); + CHECK(mod->set(ips_regex->name, vb, nullptr)); +} + +TEST(ips_regex_module, config_fail) +{ + Value vs("[[:fubar:]]"); + const Parameter* p = get_param(mod, "~"); + CHECK(p); + vs.set(p); + CHECK(mod->set(ips_regex->name, vs, nullptr)); + + Parameter bad { "bad", Parameter::PT_STRING, nullptr, nullptr, "bad" }; + vs.set(&bad); + CHECK(!mod->set(ips_regex->name, vs, nullptr)); + + expect = 1; + end = false; +} + +//------------------------------------------------------------------------- +// option tests +//------------------------------------------------------------------------- + +TEST_GROUP(ips_regex_option) +{ + IpsOption* opt = nullptr; + + void setup() + { + IpsApi* api = (IpsApi*)ips_regex; + opt = get_option("\\bfoo"); + api->tinit(nullptr); + } + void teardown() + { + IpsApi* api = (IpsApi*)ips_regex; + api->dtor(opt); + api->tterm(nullptr); + } +}; + +TEST(ips_regex_option, hash) +{ + IpsOption* opt2 = get_option("bar"); + CHECK(opt2); + CHECK(*opt != *opt2); + + uint32_t h1 = opt->hash(); + uint32_t h2 = opt2->hash(); + CHECK(h1 != h2); + + IpsApi* api = (IpsApi*)ips_regex; + api->dtor(opt2); +} + +TEST(ips_regex_option, opeq) +{ + IpsOption* opt2 = get_option("\\bfoo"); + CHECK(opt2); + CHECK(*opt == *opt2); + + IpsApi* api = (IpsApi*)ips_regex; + api->dtor(opt2); +} + +TEST(ips_regex_option, match_absolute) +{ + Packet pkt; + pkt.data = (uint8_t*)"* foo stew *"; + pkt.dsize = strlen((char*)pkt.data); + + Cursor c(&pkt); + CHECK(opt->eval(c, &pkt) == DETECTION_OPTION_MATCH); + CHECK(!strcmp((char*)c.start(), " stew *")); +} + +TEST(ips_regex_option, no_match_delta) +{ + Packet pkt; + pkt.data = (uint8_t*)"* foo stew *"; + pkt.dsize = strlen((char*)pkt.data); + + Cursor c(&pkt); + c.set_delta(3); + + CHECK(opt->eval(c, &pkt) == DETECTION_OPTION_NO_MATCH); +} + +//------------------------------------------------------------------------- +// relative tests +//------------------------------------------------------------------------- + +TEST_GROUP(ips_regex_option_relative) +{ + IpsOption* opt = nullptr; + + void setup() + { + IpsApi* api = (IpsApi*)ips_regex; + opt = get_option("\\bfoo", true); + api->tinit(nullptr); + } + void teardown() + { + IpsApi* api = (IpsApi*)ips_regex; + api->dtor(opt); + api->tterm(nullptr); + } +}; + +TEST(ips_regex_option_relative, no_match) +{ + Packet pkt; + pkt.data = (uint8_t*)"* foo stew *"; + pkt.dsize = strlen((char*)pkt.data); + + Cursor c(&pkt); + c.add_pos(3); + + CHECK(opt->is_relative()); + CHECK(opt->eval(c, &pkt) == DETECTION_OPTION_NO_MATCH); +} + +//------------------------------------------------------------------------- +// main +//------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} +