From: Russ Combs (rucombs) Date: Mon, 20 Jan 2020 14:01:54 +0000 (+0000) Subject: Merge pull request #1943 in SNORT/snort3 from ~RUCOMBS/snort3:dinty_moore to master X-Git-Tag: 3.0.0-268~49 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=59d5187887272c9c53f5cbab23d93a4d1492c83b;p=thirdparty%2Fsnort3.git Merge pull request #1943 in SNORT/snort3 from ~RUCOMBS/snort3:dinty_moore to master Squashed commit of the following: commit 505d59a649bcef812ceaa2c5656fa5d90c98775f Author: russ Date: Wed Jan 1 21:00:42 2020 -0500 detection: add pcre_override to enable/disable pcre/O commit 264b6283af3f57198ced03b3db995dbd25edf12b Author: russ Date: Wed Jan 1 20:15:54 2020 -0500 detection: add hyperscan_literals option commit b7cfac2065b0332d70b767cba705e21a9e24dc92 Author: russ Date: Mon Dec 9 04:41:41 2019 -0500 search_engine: trivial reformatting commit 00ad9c9f9c0b5db81cfc28697a6f27a24e022278 Author: russ Date: Sat Dec 14 08:40:52 2019 -0500 detection: signature evaluation looping based on literal contents only (exclude regex) commit 3fc421f1739a45bfc37bde0732c442b78386a7cc Author: russ Date: Thu Jan 2 08:10:00 2020 -0500 content: use hs_compile if hs_compile_lit is not available commit d5c5c32b613011286d0dca879ce1b92b4853d590 Author: russ Date: Mon Dec 9 04:44:15 2019 -0500 content: add hyperscan content literal matching alternative to boyer-moore commit 00feeb9b5e7dadac2ccfc9c4332bf9a7606ae075 Author: russ Date: Tue Dec 10 09:32:57 2019 -0500 framework: introduce ScratchAllocator class to help with scratch memory management commit b91dfca84c0b4bc8e8b3d686ae83f9eff1bc06d8 Author: russ Date: Sun Dec 29 08:59:21 2019 -0500 pcre: ensure use of maximal ovector size and simplify logic commit 6cd139d98726d5c058bb5733a74f9eb48879a2e7 Author: russ Date: Mon Dec 9 04:40:37 2019 -0500 hyperscan: convert thread locals to scan context commit 1a059cea1ba0c9f101039c36f1703308b946686b Author: russ Date: Mon Dec 9 04:39:48 2019 -0500 regex: convert thread locals to scan context --- diff --git a/cmake/sanity_checks.cmake b/cmake/sanity_checks.cmake index 8f2474fec..7060eaf47 100644 --- a/cmake/sanity_checks.cmake +++ b/cmake/sanity_checks.cmake @@ -132,6 +132,13 @@ endif() # set library variables if (HS_FOUND) check_library_exists (${HS_LIBRARIES} hs_scan "" HAVE_HYPERSCAN) + if (HAVE_HYPERSCAN) + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_INCLUDES ${HS_INCLUDE_DIRS}) + set(CMAKE_REQUIRED_LIBRARIES ${HS_LIBRARIES}) + check_function_exists(hs_compile_lit HAVE_HS_COMPILE_LIT) + cmake_pop_check_state() + endif() endif() if (DEFINED LIBLZMA_LIBRARIES) diff --git a/config.cmake.h.in b/config.cmake.h.in index c355f4aa4..d6330bc21 100644 --- a/config.cmake.h.in +++ b/config.cmake.h.in @@ -115,6 +115,7 @@ /* hyperscan available */ #cmakedefine HAVE_HYPERSCAN 1 +#cmakedefine HAVE_HS_COMPILE_LIT 1 /* lzma available */ #cmakedefine HAVE_LZMA 1 diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index bf374e465..d98619090 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -404,7 +404,7 @@ int detection_option_node_evaluate( IpsOption* opt = (IpsOption*)node->option_data; PatternMatchData* pmd = opt->get_pattern(0, RULE_WO_DIR); - if ( pmd and pmd->last_check ) + if ( pmd and pmd->is_literal() and pmd->last_check ) content_last = pmd->last_check + get_instance_id(); } @@ -610,7 +610,7 @@ int detection_option_node_evaluate( IpsOption* opt = (IpsOption*)child_node->option_data; PatternMatchData* pmd = opt->get_pattern(0, RULE_WO_DIR); - if ( pmd and pmd->is_unbounded() ) + if ( pmd and pmd->is_literal() and pmd->is_unbounded() ) { // Only increment result once. Should hit this // condition on first loop iteration diff --git a/src/helpers/CMakeLists.txt b/src/helpers/CMakeLists.txt index 2795baa2d..0503609d4 100644 --- a/src/helpers/CMakeLists.txt +++ b/src/helpers/CMakeLists.txt @@ -1,11 +1,28 @@ +if ( HAVE_HYPERSCAN ) + set(HYPER_HEADERS + hyper_scratch_allocator.h + hyper_search.h + ) + set(HYPER_SOURCES + hyper_scratch_allocator.cc + hyper_search.cc + ) +endif () + set (HELPERS_INCLUDES + ${HYPER_HEADERS} base64_encoder.h + boyer_moore_search.h + literal_search.h + scratch_allocator.h ) add_library (helpers OBJECT ${HELPERS_INCLUDES} + ${HYPER_SOURCES} base64_encoder.cc + boyer_moore_search.cc chunk.cc chunk.h directory.cc @@ -13,12 +30,14 @@ add_library (helpers OBJECT discovery_filter.cc discovery_filter.h flag_context.h + literal_search.cc markup.cc markup.h process.cc process.h ring.h ring_logic.h + scratch_allocator.cc ) install (FILES ${HELPERS_INCLUDES} @@ -30,3 +49,6 @@ add_catch_test( base64_encoder_test SOURCES base64_encoder.cc ) + +add_subdirectory(test) + diff --git a/src/helpers/boyer_moore_search.cc b/src/helpers/boyer_moore_search.cc new file mode 100644 index 000000000..7f4b16ded --- /dev/null +++ b/src/helpers/boyer_moore_search.cc @@ -0,0 +1,89 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// boyer_moore_search.cc author Brandon Stultz + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "boyer_moore_search.h" + +namespace snort +{ + +BoyerMooreSearch::BoyerMooreSearch(const uint8_t* pattern, unsigned pattern_len) + : pattern(pattern), pattern_len(pattern_len) +{ + assert(pattern_len > 0); + + last = pattern_len - 1; + + make_skip(); +} + +// skip[c] is the distance between the last character of the +// pattern and the rightmost occurrence of c in the pattern. +// If c does not occur in the pattern then skip[c] = pattern_len. +void BoyerMooreSearch::make_skip() +{ + for ( unsigned i = 0; i < 256; i++ ) + skip[i] = pattern_len; + + for ( unsigned i = 0; i < last; i++ ) + skip[pattern[i]] = last - i; +} + +int BoyerMooreSearchCase::search(const uint8_t* buffer, unsigned buffer_len) const +{ + const uint8_t* start = buffer; + + while ( buffer_len >= pattern_len ) + { + for ( unsigned pos = last; buffer[pos] == pattern[pos]; pos-- ) + if ( pos == 0 ) + return buffer - start; + + buffer_len -= skip[buffer[last]]; + buffer += skip[buffer[last]]; + } + + return -1; +} + +int BoyerMooreSearchNoCase::search(const uint8_t* buffer, unsigned buffer_len) const +{ + const uint8_t* start = buffer; + + while ( buffer_len >= pattern_len ) + { + for ( unsigned pos = last; toupper(buffer[pos]) == pattern[pos]; pos-- ) + if ( pos == 0 ) + return buffer - start; + + buffer_len -= skip[toupper(buffer[last])]; + buffer += skip[toupper(buffer[last])]; + } + + return -1; +} + +} + diff --git a/src/helpers/boyer_moore_search.h b/src/helpers/boyer_moore_search.h new file mode 100644 index 000000000..837472031 --- /dev/null +++ b/src/helpers/boyer_moore_search.h @@ -0,0 +1,73 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// boyer_moore_search.h author Brandon Stultz + +#ifndef BOYER_MOORE_SEARCH_H +#define BOYER_MOORE_SEARCH_H + +// Boyer-Moore literal content matching routines (single pattern) +// use LiteralSearch::instantiate to get hyperscan if available + +#include "helpers/literal_search.h" +#include "main/snort_types.h" + +namespace snort +{ + +class SO_PUBLIC BoyerMooreSearch : public LiteralSearch +{ +protected: + BoyerMooreSearch(const uint8_t* pattern, unsigned pattern_len); + +protected: + void make_skip(); + + const uint8_t* pattern; + unsigned pattern_len; + unsigned last; + + unsigned skip[256]; +}; + +class SO_PUBLIC BoyerMooreSearchCase : public BoyerMooreSearch +{ +public: + BoyerMooreSearchCase(const uint8_t* pat, unsigned pat_len) : + BoyerMooreSearch(pat, pat_len) { } + + int search(const uint8_t* buffer, unsigned buffer_len) const; + + int search(void*, const uint8_t* buffer, unsigned buffer_len) const override + { return search(buffer, buffer_len); } +}; + +class SO_PUBLIC BoyerMooreSearchNoCase : public BoyerMooreSearch +{ +public: + BoyerMooreSearchNoCase(const uint8_t* pat, unsigned pat_len) : + BoyerMooreSearch(pat, pat_len) { } + + int search(const uint8_t* buffer, unsigned buffer_len) const; + + int search(void*, const uint8_t* buffer, unsigned buffer_len) const override + { return search(buffer, buffer_len); } +}; + +} +#endif + diff --git a/src/helpers/hyper_scratch_allocator.cc b/src/helpers/hyper_scratch_allocator.cc new file mode 100644 index 000000000..66f56888e --- /dev/null +++ b/src/helpers/hyper_scratch_allocator.cc @@ -0,0 +1,69 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// hyper_scratch_allocator.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "hyper_scratch_allocator.h" +#include "log/messages.h" + +namespace snort +{ + +HyperScratchAllocator::~HyperScratchAllocator() +{ + if ( scratch ) + hs_free_scratch(scratch); +} + +bool HyperScratchAllocator::allocate(hs_database_t* db) +{ + if ( scratch_id < 0 ) + scratch_id = SnortConfig::request_scratch(this); + + return hs_alloc_scratch(db, &scratch) == HS_SUCCESS; +} + +bool HyperScratchAllocator::setup(SnortConfig* sc) +{ + if ( !scratch ) + return false; + + for ( unsigned i = 0; i < sc->num_slots; ++i ) + hs_clone_scratch(scratch, get_addr(sc, i)); + + hs_free_scratch(scratch); + scratch = nullptr; + + return true; +} + +void HyperScratchAllocator::cleanup(SnortConfig* sc) +{ + for ( unsigned i = 0; i < sc->num_slots; ++i ) + { + hs_scratch_t* ss = get(sc, i); + hs_free_scratch(ss); + set(sc, i, nullptr); + } +} + +} + diff --git a/src/helpers/hyper_scratch_allocator.h b/src/helpers/hyper_scratch_allocator.h new file mode 100644 index 000000000..3fa245165 --- /dev/null +++ b/src/helpers/hyper_scratch_allocator.h @@ -0,0 +1,70 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// hyper_scratch_allocator.h author Russ Combs + +#ifndef HYPER_SCRATCH_ALLOCATOR_H +#define HYPER_SCRATCH_ALLOCATOR_H + +#include "helpers/scratch_allocator.h" + +#include +#include + +#include "main/snort_config.h" +#include "main/snort_types.h" +#include "main/thread.h" + +//-------------------------------------------------------------------------- +// scratch management +//-------------------------------------------------------------------------- + +namespace snort +{ +struct SnortConfig; + +class SO_PUBLIC HyperScratchAllocator : public ScratchAllocator +{ +public: + ~HyperScratchAllocator() override; + + bool setup(SnortConfig*) override; + void cleanup(SnortConfig*) override; + + bool allocate(hs_database_t*); + + hs_scratch_t* get() + { return get(SnortConfig::get_conf(), snort::get_instance_id()); } + +private: + hs_scratch_t** get_addr(SnortConfig* sc, unsigned idx) + { return (hs_scratch_t**)&sc->state[idx][scratch_id]; } + + hs_scratch_t* get(SnortConfig* sc, unsigned idx) + { return (hs_scratch_t*)sc->state[idx][scratch_id]; } + + void set(SnortConfig* sc, unsigned idx, void* pv) + { sc->state[idx][scratch_id] = pv; } + +private: + hs_scratch_t* scratch = nullptr; + int scratch_id = -1; +}; + +} +#endif + diff --git a/src/helpers/hyper_search.cc b/src/helpers/hyper_search.cc new file mode 100644 index 000000000..d13f6444a --- /dev/null +++ b/src/helpers/hyper_search.cc @@ -0,0 +1,126 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// hyper_search.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "hyper_search.h" + +#include +#include +#include + +#include +#include + +#include "log/messages.h" +#include "main/snort_config.h" +#include "main/thread.h" + +#include "hyper_scratch_allocator.h" + +namespace snort +{ + +LiteralSearch::Handle* HyperSearch::setup() +{ return new HyperScratchAllocator; } + +void HyperSearch::cleanup(LiteralSearch::Handle* h) +{ + HyperScratchAllocator* scratcher = (HyperScratchAllocator*)h; + delete scratcher; +} + +//-------------------------------------------------------------------------- + +HyperSearch::HyperSearch(LiteralSearch::Handle* h, const uint8_t* pattern, unsigned len, bool no_case) +{ + assert(h); + HyperScratchAllocator* scratcher = (HyperScratchAllocator*)h; + + assert(len > 0); + pattern_len = len; + + hs_compile_error_t* err = nullptr; + + unsigned flags = HS_FLAG_SINGLEMATCH; + if ( no_case ) + flags |= HS_FLAG_CASELESS; + +#ifndef HAVE_HS_COMPILE_LIT + std::string hex_pat; + + for ( unsigned i = 0; i < len; ++i ) + { + char hex[5]; + snprintf(hex, sizeof(hex), "\\x%02X", pattern[i]); + hex_pat += hex; + } + + if ( hs_compile((const char*)hex_pat.c_str(), flags, + HS_MODE_BLOCK, nullptr, (hs_database_t**)&db, &err) != HS_SUCCESS ) +#else + if ( hs_compile_lit((const char*)pattern, flags, pattern_len, + HS_MODE_BLOCK, nullptr, (hs_database_t**)&db, &err) != HS_SUCCESS ) +#endif + { + ParseError("can't compile content '%s'", pattern); + hs_free_compile_error(err); + return; + } + if ( !scratcher->allocate(db) ) + ParseError("can't allocate scratch for content '%s'", pattern); +} + +HyperSearch::~HyperSearch() +{ + if ( db ) + hs_free_database(db); +} + +} + +struct ScanContext +{ + unsigned index; + bool found = false; +}; + +static int hs_match(unsigned int, unsigned long long, unsigned long long to, unsigned int, void* context) +{ + ScanContext* scan = (ScanContext*)context; + scan->index = (unsigned)to; + scan->found = true; + return 1; +} + +namespace snort +{ + +int HyperSearch::search(LiteralSearch::Handle* h, const uint8_t* buffer, unsigned buffer_len) const +{ + HyperScratchAllocator* scratcher = (HyperScratchAllocator*)h; + ScanContext scan; + hs_scan(db, (const char*)buffer, buffer_len, 0, scratcher->get(), hs_match, &scan); + return scan.found ? ((int)(scan.index - pattern_len)) : -1; +} + +} + diff --git a/src/helpers/hyper_search.h b/src/helpers/hyper_search.h new file mode 100644 index 000000000..69304bbe7 --- /dev/null +++ b/src/helpers/hyper_search.h @@ -0,0 +1,55 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// hyper_search.h author Russ Combs + +#ifndef HYPER_SEARCH_H +#define HYPER_SEARCH_H + +// Hyperscan-based literal content matching (single pattern) +// use LiteralSearch::instantiate to fallback to boyer-moore +// if hyperscan is not available. + +#include "helpers/literal_search.h" +#include "main/snort_types.h" + +extern "C" struct hs_database; + +namespace snort +{ + +class SO_PUBLIC HyperSearch : public snort::LiteralSearch +{ +public: + using Handle = snort::LiteralSearch::Handle; + + static Handle* setup(); // call from module ctor + static void cleanup(Handle*); // call from module dtor + + HyperSearch(Handle*, const uint8_t* pattern, unsigned pattern_len, bool no_case = false); + ~HyperSearch() override; + + int search(Handle*, const uint8_t* buffer, unsigned buffer_len) const override; + +private: + struct hs_database* db; + unsigned pattern_len; +}; + +} +#endif + diff --git a/src/helpers/literal_search.cc b/src/helpers/literal_search.cc new file mode 100644 index 000000000..4738ab293 --- /dev/null +++ b/src/helpers/literal_search.cc @@ -0,0 +1,72 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// literal_search.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "literal_search.h" + +#include + +#include "main/snort_config.h" +#include "boyer_moore_search.h" +#include "hyper_search.h" + +namespace snort +{ + +// setup and cleanup for hyperscan are independent of configuration +// because that would create a bad dependency - a module ctor needs +// a config item from a module. also, the handle must persist for +// for the lifetime of a module, which can span many configs. + +LiteralSearch::Handle* LiteralSearch::setup() +{ +#ifdef HAVE_HYPERSCAN + return HyperSearch::setup(); +#else + return nullptr; +#endif +} + +void LiteralSearch::cleanup(LiteralSearch::Handle* h) +{ +#ifdef HAVE_HYPERSCAN + HyperSearch::cleanup(h); +#endif +} + +LiteralSearch* LiteralSearch::instantiate( + LiteralSearch::Handle* h, const uint8_t* pattern, unsigned pattern_len, bool no_case) +{ +#ifdef HAVE_HYPERSCAN + if ( SnortConfig::get_conf()->hyperscan_literals ) + return new HyperSearch(h, pattern, pattern_len, no_case); +#else + UNUSED(h); +#endif + if ( no_case ) + return new snort::BoyerMooreSearchNoCase(pattern, pattern_len); + + return new snort::BoyerMooreSearchCase(pattern, pattern_len); +} + +} + diff --git a/src/helpers/literal_search.h b/src/helpers/literal_search.h new file mode 100644 index 000000000..1dc4c8886 --- /dev/null +++ b/src/helpers/literal_search.h @@ -0,0 +1,50 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// literal_search.h author Russ Combs + +#ifndef LITERAL_SEARCH_H +#define LITERAL_SEARCH_H + +// literal content matching (single pattern) +// used eg with content during signature evaluation + +#include "main/snort_types.h" + +namespace snort +{ + +class SO_PUBLIC LiteralSearch +{ +public: + using Handle = void; + + static Handle* setup(); // call from module ctor + static void cleanup(Handle*); // call from module dtor + + static LiteralSearch* instantiate(Handle*, const uint8_t* pattern, unsigned pattern_len, bool no_case = false); + virtual ~LiteralSearch() { } + + virtual int search(Handle*, const uint8_t* buffer, unsigned buffer_len) const = 0; + +protected: + LiteralSearch() { } +}; + +} +#endif + diff --git a/src/helpers/scratch_allocator.cc b/src/helpers/scratch_allocator.cc new file mode 100644 index 000000000..4896de2de --- /dev/null +++ b/src/helpers/scratch_allocator.cc @@ -0,0 +1,37 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// scratch_allocator.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "scratch_allocator.h" +#include "main/snort_config.h" + +namespace snort +{ + +ScratchAllocator::ScratchAllocator() +{ id = SnortConfig::request_scratch(this); } + +ScratchAllocator::~ScratchAllocator() +{ SnortConfig::release_scratch(id); } + +} + diff --git a/src/helpers/scratch_allocator.h b/src/helpers/scratch_allocator.h new file mode 100644 index 000000000..e416dab6a --- /dev/null +++ b/src/helpers/scratch_allocator.h @@ -0,0 +1,82 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// scratch_allocator.h author Russ Combs + +#ifndef SCRATCH_ALLOCATOR_H +#define SCRATCH_ALLOCATOR_H + +// manages scratch memory - allocates required memory for each packet thread +// in SnortConfig.state[slot][id] where 0 <= slot < SnortConfig.num_slots and +// id = SnortConfig::request_scratch(). The use of scratch memory is strictly +// per packet, it can not be referenced on a flow, as it will change with each +// config reload. +// +// setup() should return false if no memory was allocated otherwise cleanup() +// will be called when the config is deleted. this can happen eg if the +// associated module is not used in the current configuration. +// +// scratch allocators may use a prototype to allocate the packet thread +// memory. the prototype should be freed in setup to avoid leaks and to +// ensure the prototypes for different configs are not interdependent (eg +// preventing a decrease in required scratch). + +#include "main/snort_types.h" + +namespace snort +{ +struct SnortConfig; + +class SO_PUBLIC ScratchAllocator +{ +public: + virtual ~ScratchAllocator(); + + virtual bool setup(SnortConfig*) = 0; + virtual void cleanup(SnortConfig*) = 0; + + int get_id() { return id; } + +protected: + ScratchAllocator(); + +private: + int id; +}; + +typedef bool (* ScratchSetup)(SnortConfig*); +typedef void (* ScratchCleanup)(SnortConfig*); + +class SO_PUBLIC SimpleScratchAllocator : public ScratchAllocator +{ +public: + SimpleScratchAllocator(ScratchSetup fs, ScratchCleanup fc) : fsetup(fs), fcleanup(fc) { } + + bool setup(SnortConfig* sc) override + { return fsetup(sc); } + + void cleanup(SnortConfig* sc) override + { fcleanup(sc); } + +private: + ScratchSetup fsetup; + ScratchCleanup fcleanup; +}; + +} +#endif + diff --git a/src/helpers/test/CMakeLists.txt b/src/helpers/test/CMakeLists.txt new file mode 100644 index 000000000..07e1b103d --- /dev/null +++ b/src/helpers/test/CMakeLists.txt @@ -0,0 +1,16 @@ +add_cpputest( boyer_moore_search_test + SOURCES + ../boyer_moore_search.cc +) + +if ( HAVE_HYPERSCAN ) + add_cpputest( hyper_search_test + SOURCES + ../hyper_search.cc + ../../helpers/scratch_allocator.cc + ../../helpers/hyper_scratch_allocator.cc + LIBS + ${HS_LIBRARIES} + ) +endif() + diff --git a/src/helpers/test/boyer_moore_search_test.cc b/src/helpers/test/boyer_moore_search_test.cc new file mode 100644 index 000000000..123b528f3 --- /dev/null +++ b/src/helpers/test/boyer_moore_search_test.cc @@ -0,0 +1,195 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// boyer_moore_search_test.cc author Brandon Stultz + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "../boyer_moore_search.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace snort; + +enum TestType +{ + CASE, + NOCASE, +}; + +class Tester +{ +public: + Tester(const char* pat_str, const char* buf_str, TestType typ, int idx) + : pat(pat_str), buf(buf_str), type(typ), index(idx) + { + if ( type == NOCASE ) + transform(pat.begin(), pat.end(), pat.begin(), ::toupper); + + pattern_len = pat.length(); + buffer_len = buf.length(); + + pattern = (const uint8_t*)(pat.c_str()); + buffer = (const uint8_t*)(buf.c_str()); + } + + bool run(); + +private: + string pat; + string buf; + + TestType type; + + int index; + + unsigned pattern_len; + unsigned buffer_len; + + const uint8_t* pattern; + const uint8_t* buffer; +}; + +bool Tester::run() +{ + int pos; + + if ( type == NOCASE ) + { + BoyerMooreSearchNoCase bm = BoyerMooreSearchNoCase(pattern, pattern_len); + pos = bm.search(buffer, buffer_len); + } + else + { + BoyerMooreSearchCase bm = BoyerMooreSearchCase(pattern, pattern_len); + pos = bm.search(buffer, buffer_len); + } + + return pos == index; +} + +TEST_GROUP(boyer_moore_test_group) {}; + +TEST(boyer_moore_test_group, binary) +{ + const uint8_t pat[] = { 0xCA, 0xFE, 0xBA, 0xBE }; + + const uint8_t buf[] = { + 0x00, 0x01, 0x02, 0x03, + 0x72, 0x01, 0x3F, 0x2B, + 0x1F, 0xCA, 0xFE, 0xBA, + 0xBE, 0x01, 0x02, 0x03, + }; + + BoyerMooreSearchCase bm = BoyerMooreSearchCase(pat, sizeof(pat)); + + int pos = bm.search(buf, sizeof(buf)); + + CHECK(pos == 9); +} + +TEST(boyer_moore_test_group, empty) +{ + Tester t = Tester("abc", "", CASE, -1); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, start) +{ + Tester t = Tester("abc", "abc", CASE, 0); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, start_nocase) +{ + Tester t = Tester("abc", "aBc", NOCASE, 0); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found1) +{ + Tester t = Tester("d", "abcdefg", CASE, 3); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found2) +{ + Tester t = Tester("nan", "banana", CASE, 2); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found3) +{ + Tester t = Tester("pan", "anpanman", CASE, 2); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found4) +{ + Tester t = Tester("bcd", "abcd", CASE, 1); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found5) +{ + Tester t = Tester("aa", "aaa", CASE, 0); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, found6) +{ + Tester t = Tester( + "that", "which finally halts at tHaT point", NOCASE, 23); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, not_found1) +{ + Tester t = Tester("nnaaman", "anpanmanam", CASE, -1); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, not_found2) +{ + Tester t = Tester("abcd", "abc", CASE, -1); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, not_found3) +{ + Tester t = Tester("abcd", "bcd", CASE, -1); + CHECK(t.run()); +} + +TEST(boyer_moore_test_group, not_found4) +{ + Tester t = Tester("baa", "aaaaa", CASE, -1); + CHECK(t.run()); +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/helpers/test/hyper_search_test.cc b/src/helpers/test/hyper_search_test.cc new file mode 100644 index 000000000..eb0f08e9c --- /dev/null +++ b/src/helpers/test/hyper_search_test.cc @@ -0,0 +1,266 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// hyper_search_test.cc author Russ Combs +// with tests ripped from boyer_moore_search_test.cc + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "../hyper_search.h" + +#include "main/snort_config.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace snort; + +//------------------------------------------------------------------------- +// mock snort config scratch handling +//------------------------------------------------------------------------- + +namespace snort +{ + +SnortConfig s_conf; +THREAD_LOCAL SnortConfig* snort_conf = &s_conf; + +static std::vector s_state; +static ScratchAllocator* scratcher = nullptr; + +static unsigned s_parse_errors = 0; + +SnortConfig::SnortConfig(const SnortConfig* const) +{ + state = &s_state; + num_slots = 1; +} + +SnortConfig::~SnortConfig() = default; + +int SnortConfig::request_scratch(ScratchAllocator* s) +{ + scratcher = s; + s_state.resize(1); + return 0; +} + +void SnortConfig::release_scratch(int) +{ s_state.clear(); } + +SnortConfig* SnortConfig::get_conf() +{ return snort_conf; } + +void ParseError(const char*, ...) +{ ++s_parse_errors; } + +unsigned get_instance_id() +{ return 0; } + +} + +//------------------------------------------------------------------------- +// tests +//------------------------------------------------------------------------- + +enum TestType +{ + CASE, + NOCASE, +}; + +class Tester +{ +public: + Tester(const char* pat_str, const char* buf_str, TestType typ, int idx) + : pat(pat_str), buf(buf_str), type(typ), index(idx) + { + if ( type == NOCASE ) + transform(pat.begin(), pat.end(), pat.begin(), ::toupper); + + pattern_len = pat.length(); + buffer_len = buf.length(); + + pattern = (const uint8_t*)(pat.c_str()); + buffer = (const uint8_t*)(buf.c_str()); + } + + bool run(HyperSearch::Handle*); + +private: + string pat; + string buf; + + TestType type; + + int index; + + unsigned pattern_len; + unsigned buffer_len; + + const uint8_t* pattern; + const uint8_t* buffer; +}; + +bool Tester::run(HyperSearch::Handle* handle) +{ + HyperSearch hs(handle, pattern, pattern_len, (type == NOCASE)); + bool do_cleanup = scratcher->setup(snort_conf); + int pos = hs.search(handle, buffer, buffer_len); + + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + + return pos == index; +} + +TEST_GROUP(hyper_search_test_group) +{ + HyperSearch::Handle* handle = nullptr; + + void setup() override + { + s_parse_errors = 0; + handle = HyperSearch::setup(); + } + + void teardown() override + { + HyperSearch::cleanup(handle); + CHECK(s_parse_errors == 0); + } +}; + +TEST(hyper_search_test_group, binary) +{ + const uint8_t pat[] = { 0xCA, 0xFE, 0xBA, 0xBE, 0x00 }; + + const uint8_t buf[] = { + 0x00, 0x01, 0x02, 0x03, + 0x72, 0x01, 0x3F, 0x2B, + 0x1F, 0xCA, 0xFE, 0xBA, + 0xBE, 0x01, 0x02, 0x03, + }; + + HyperSearch hs(handle, pat, sizeof(pat)-1, true); + bool do_cleanup = scratcher->setup(snort_conf); + int pos = hs.search(handle, buf, sizeof(buf)); + + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + + CHECK(pos == 9); +} + +TEST(hyper_search_test_group, empty) +{ + Tester t = Tester("abc", "", CASE, -1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, start) +{ + Tester t = Tester("abc", "abc", CASE, 0); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, start_nocase) +{ + Tester t = Tester("abc", "aBc", NOCASE, 0); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found1) +{ + Tester t = Tester("d", "abcdefg", CASE, 3); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found2) +{ + Tester t = Tester("nan", "banana", CASE, 2); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found3) +{ + Tester t = Tester("pan", "anpanman", CASE, 2); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found4) +{ + Tester t = Tester("bcd", "abcd", CASE, 1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found5) +{ + Tester t = Tester("aa", "aaa", CASE, 0); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, found6) +{ + Tester t = Tester( + "that", "which finally halts at tHaT point", NOCASE, 23); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, not_found1) +{ + Tester t = Tester("nnaaman", "anpanmanam", CASE, -1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, not_found2) +{ + Tester t = Tester("abcd", "abc", CASE, -1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, not_found3) +{ + Tester t = Tester("abcd", "bcd", CASE, -1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, not_found4) +{ + Tester t = Tester("baa", "aaaaa", CASE, -1); + CHECK(t.run(handle)); +} + +TEST(hyper_search_test_group, compiler_error) +{ + HyperSearch hs(handle, nullptr, 1); + CHECK(s_parse_errors == 1); + s_parse_errors = 0; +} + +int main(int argc, char** argv) +{ + MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc index 0edc0568f..96abbe226 100644 --- a/src/ips_options/ips_content.cc +++ b/src/ips_options/ips_content.cc @@ -27,10 +27,10 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashfcn.h" +#include "helpers/literal_search.h" #include "log/messages.h" #include "parser/parse_utils.h" #include "profiler/profiler.h" -#include "utils/boyer_moore.h" #include "utils/util.h" #include "utils/stats.h" @@ -43,6 +43,7 @@ using namespace snort; #define s_name "content" static THREAD_LOCAL ProfileStats contentPerfStats; +static LiteralSearch::Handle* search_handle = nullptr; static IpsOption::EvalStatus CheckANDPatternMatch(class ContentData*, Cursor&); @@ -54,7 +55,6 @@ class ContentData { public: ContentData(); - ~ContentData(); void setup_bm(); @@ -62,7 +62,7 @@ public: PatternMatchData pmd = {}; - BoyerMoore* boyer_moore; + LiteralSearch* searcher; int8_t offset_var; /* byte_extract variable indices for offset, */ int8_t depth_var; /* depth, distance, within */ @@ -72,7 +72,7 @@ public: ContentData::ContentData() { - boyer_moore = nullptr; + searcher = nullptr; offset_var = IPS_OPTIONS_NO_VAR; depth_var = IPS_OPTIONS_NO_VAR; match_delta = 0; @@ -80,8 +80,8 @@ ContentData::ContentData() ContentData::~ContentData() { - if ( boyer_moore ) - delete boyer_moore; + if ( searcher ) + delete searcher; if ( pmd.pattern_buf ) snort_free(const_cast(pmd.pattern_buf)); @@ -93,8 +93,7 @@ ContentData::~ContentData() void ContentData::setup_bm() { const uint8_t* pattern = (const uint8_t*)pmd.pattern_buf; - - boyer_moore = new BoyerMoore(pattern, pmd.pattern_size); + searcher = LiteralSearch::instantiate(search_handle, pattern, pmd.pattern_size, pmd.is_no_case()); } // find the maximum number of characters we can jump ahead @@ -345,16 +344,7 @@ static int uniSearchReal(ContentData* cd, Cursor& c) } const uint8_t* base = c.buffer() + pos; - int found; - - if ( cd->pmd.is_no_case() ) - { - found = cd->boyer_moore->search_nocase(base, depth); - } - else - { - found = cd->boyer_moore->search(base, depth); - } + int found = cd->searcher->search(search_handle, base, depth); if ( found >= 0 ) { @@ -634,10 +624,16 @@ class ContentModule : public Module { public: ContentModule() : Module(s_name, s_help, s_params) - { cd = nullptr; } + { + cd = nullptr; + search_handle = LiteralSearch::setup(); + } ~ContentModule() override - { delete cd; } + { + delete cd; + LiteralSearch::cleanup(search_handle); + } bool begin(const char*, int, SnortConfig*) override; bool end(const char*, int, SnortConfig*) override; diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index 4a9450c6f..83b9abc7d 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -31,6 +31,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashfcn.h" +#include "helpers/scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" #include "profiler/profiler.h" @@ -68,25 +69,17 @@ struct PcreData char* expression; }; -/* - * we need to specify the vector length for our pcre_exec call. we only care - * about the first vector, which if the match is successful will include the - * offset to the end of the full pattern match. If we decide to store other - * matches, make *SURE* that this is a multiple of 3 as pcre requires it. - */ -// the wrong size caused the pcre lib to segfault but that has since been -// fixed. it may be that with the updated lib, the need to get the size -// exactly correct is obviated and thus the need to reload as well. - -/* Since SO rules are loaded 1 time at startup, regardless of - * configuration, we won't pcre_capture count again, so save the max. */ -static int s_ovector_max = 0; +// we need to specify the vector length for our pcre_exec call. we only care +// about the first vector, which if the match is successful will include the +// offset to the end of the full pattern match. if we decide to store other +// matches, make *SURE* that this is a multiple of 3 as pcre requires it. // this is a temporary value used during parsing and set in snort conf // by verify; search uses the value in snort conf -static int s_ovector_size = 0; +static int s_ovector_max = -1; static unsigned scratch_index; +static ScratchAllocator* scratcher = nullptr; static THREAD_LOCAL ProfileStats pcrePerfStats; @@ -102,8 +95,8 @@ static void pcre_capture( pcre_fullinfo((const pcre*)code, (const pcre_extra*)extra, PCRE_INFO_CAPTURECOUNT, &tmp_ovector_size); - if (tmp_ovector_size > s_ovector_size) - s_ovector_size = tmp_ovector_size; + if (tmp_ovector_size > s_ovector_max) + s_ovector_max = tmp_ovector_size; } static void pcre_check_anchored(PcreData* pcre_data) @@ -156,7 +149,7 @@ static void pcre_check_anchored(PcreData* pcre_data) } } -static void pcre_parse(const char* data, PcreData* pcre_data) +static void pcre_parse(const SnortConfig* sc, const char* data, PcreData* pcre_data) { const char* error; char* re, * free_me; @@ -248,7 +241,10 @@ static void pcre_parse(const char* data, PcreData* pcre_data) * these are snort specific don't work with pcre or perl */ case 'R': pcre_data->options |= SNORT_PCRE_RELATIVE; break; - case 'O': pcre_data->options |= SNORT_OVERRIDE_MATCH_LIMIT; break; + case 'O': + if ( sc->pcre_override ) + pcre_data->options |= SNORT_OVERRIDE_MATCH_LIMIT; + break; default: ParseError("unknown/extra pcre option encountered"); @@ -619,12 +615,15 @@ public: PcreModule() : Module(s_name, s_help, s_params) { data = nullptr; - scratch_index = SnortConfig::request_scratch( - PcreModule::scratch_setup, PcreModule::scratch_cleanup); + scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup); + scratch_index = scratcher->get_id(); } ~PcreModule() override - { delete data; } + { + delete data; + delete scratcher; + } bool begin(const char*, int, SnortConfig*) override; bool set(const char*, Value&, SnortConfig*) override; @@ -639,7 +638,7 @@ public: private: PcreData* data; - static void scratch_setup(SnortConfig* sc); + static bool scratch_setup(SnortConfig* sc); static void scratch_cleanup(SnortConfig* sc); }; @@ -656,10 +655,10 @@ bool PcreModule::begin(const char*, int, SnortConfig*) return true; } -bool PcreModule::set(const char*, Value& v, SnortConfig*) +bool PcreModule::set(const char*, Value& v, SnortConfig* sc) { if ( v.is("~re") ) - pcre_parse(v.get_string(), data); + pcre_parse(sc, v.get_string(), data); else return false; @@ -667,13 +666,26 @@ bool PcreModule::set(const char*, Value& v, SnortConfig*) return true; } -void PcreModule::scratch_setup(SnortConfig* sc) +bool PcreModule::scratch_setup(SnortConfig* sc) { + if ( s_ovector_max < 0 ) + return false; + + // The pcre_fullinfo() function can be used to find out how many + // capturing subpatterns there are in a compiled pattern. The + // smallest size for ovector that will allow for n captured + // substrings, in addition to the offsets of the substring matched + // by the whole pattern is 3(n+1). + + sc->pcre_ovector_size = 3 * (s_ovector_max + 1); + s_ovector_max = -1; + for ( unsigned i = 0; i < sc->num_slots; ++i ) { std::vector& ss = sc->state[i]; - ss[scratch_index] = snort_calloc(s_ovector_max, sizeof(int)); + ss[scratch_index] = snort_calloc(sc->pcre_ovector_size, sizeof(int)); } + return true; } void PcreModule::scratch_cleanup(SnortConfig* sc) @@ -681,10 +693,7 @@ void PcreModule::scratch_cleanup(SnortConfig* sc) for ( unsigned i = 0; i < sc->num_slots; ++i ) { std::vector& ss = sc->state[i]; - - if ( ss[scratch_index] ) - snort_free(ss[scratch_index]); - + snort_free(ss[scratch_index]); ss[scratch_index] = nullptr; } } @@ -715,23 +724,6 @@ static void pcre_dtor(IpsOption* p) delete p; } -static void pcre_verify(SnortConfig* sc) -{ - /* The pcre_fullinfo() function can be used to find out how many - * capturing subpatterns there are in a compiled pattern. The - * smallest size for ovector that will allow for n captured - * substrings, in addition to the offsets of the substring matched - * by the whole pattern, is (n+1)*3. */ - s_ovector_size += 1; - s_ovector_size *= 3; - - if (s_ovector_size > s_ovector_max) - s_ovector_max = s_ovector_size; - - sc->pcre_ovector_size = s_ovector_size; - s_ovector_size = 0; -} - static const IpsApi pcre_api = { { @@ -754,7 +746,7 @@ static const IpsApi pcre_api = nullptr, pcre_ctor, pcre_dtor, - pcre_verify + nullptr }; #ifdef BUILDING_SO diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index 44cf9087f..a86f106cf 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -33,6 +33,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashfcn.h" +#include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" #include "profiler/profiler.h" @@ -61,19 +62,7 @@ struct RegexConfig } }; -// we need to update scratch in the main thread as each pattern is processed -// and then clone to thread specific after all rules are loaded. s_scratch is -// a prototype that is large enough for all uses. - -// FIXIT-L s_scratch persists for the lifetime of the program. it is -// modeled off 2X where, due to so rule processing at startup, it is necessary -// to monotonically grow the ovector. however, we should be able to free -// s_scratch after cloning since so rules are now parsed the same as text -// rules. - -static hs_scratch_t* s_scratch = nullptr; -static unsigned scratch_index; -static THREAD_LOCAL unsigned s_to = 0; +static HyperScratchAllocator* scratcher = nullptr; static THREAD_LOCAL ProfileStats regex_perf_stats; //------------------------------------------------------------------------- @@ -111,12 +100,9 @@ RegexOption::RegexOption(const RegexConfig& c) : { config = c; - if ( /*hs_error_t err =*/ hs_alloc_scratch(config.db, &s_scratch) ) - { - // FIXIT-RC why is this failing but everything is working? - //ParseError("can't initialize regex for '%s' (%d) %p", - // config.re.c_str(), err, s_scratch); - } + if ( !scratcher->allocate(config.db) ) + ParseError("can't allocate scratch for regex '%s'", config.re.c_str()); + config.pmd.pattern_buf = config.re.c_str(); config.pmd.pattern_size = config.re.size(); @@ -156,12 +142,20 @@ bool RegexOption::operator==(const IpsOption& ips) const return this == &ips; } +struct ScanContext +{ + unsigned index; + bool found = false; +}; + static int hs_match( unsigned int /*id*/, unsigned long long /*from*/, unsigned long long to, - unsigned int /*flags*/, void* /*context*/) + unsigned int /*flags*/, void* context) { - s_to = (unsigned)to; - return 1; // stop search + ScanContext* scan = (ScanContext*)context; + scan->index = (unsigned)to; + scan->found = true; + return 1; } IpsOption::EvalStatus RegexOption::eval(Cursor& c, Packet*) @@ -176,20 +170,17 @@ IpsOption::EvalStatus RegexOption::eval(Cursor& c, Packet*) if ( pos > c.size() ) return NO_MATCH; - hs_scratch_t* ss = - (hs_scratch_t*)SnortConfig::get_conf()->state[get_instance_id()][scratch_index]; - - s_to = 0; + ScanContext scan; hs_error_t stat = hs_scan( config.db, (const char*)c.buffer()+pos, c.size()-pos, 0, - ss, hs_match, nullptr); + scratcher->get(), hs_match, &scan); - if ( s_to and stat == HS_SCAN_TERMINATED ) + if ( scan.found and stat == HS_SCAN_TERMINATED ) { - s_to += pos; - c.set_pos(s_to); - c.set_delta(s_to); + scan.index += pos; + c.set_pos(scan.index); + c.set_delta(scan.index); return MATCH; } return NO_MATCH; @@ -231,10 +222,8 @@ class RegexModule : public Module { public: RegexModule() : Module(s_name, s_help, s_params) - { - scratch_index = SnortConfig::request_scratch( - RegexModule::scratch_setup, RegexModule::scratch_cleanup); - } + { scratcher = new HyperScratchAllocator; } + ~RegexModule() override; bool begin(const char*, int, SnortConfig*) override; @@ -255,14 +244,14 @@ public: private: RegexConfig config; - static void scratch_setup(SnortConfig* sc); - static void scratch_cleanup(SnortConfig* sc); }; RegexModule::~RegexModule() { if ( config.db ) hs_free_database(config.db); + + delete scratcher; } bool RegexModule::begin(const char*, int, SnortConfig*) @@ -327,33 +316,6 @@ bool RegexModule::end(const char*, int, SnortConfig*) return true; } -void RegexModule::scratch_setup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index]; - - if ( s_scratch ) - hs_clone_scratch(s_scratch, ss); - else - ss = nullptr; - } -} - -void RegexModule::scratch_cleanup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - hs_scratch_t* ss = (hs_scratch_t*) sc->state[i][scratch_index]; - - if ( ss ) - { - hs_free_scratch(ss); - ss = nullptr; - } - } -} - //------------------------------------------------------------------------- // api methods //------------------------------------------------------------------------- @@ -375,14 +337,6 @@ static IpsOption* regex_ctor(Module* m, OptTreeNode*) static void regex_dtor(IpsOption* p) { delete p; } -static void regex_pterm(SnortConfig*) -{ - if ( s_scratch ) - hs_free_scratch(s_scratch); - - s_scratch = nullptr; -} - static const IpsApi regex_api = { { @@ -400,7 +354,7 @@ static const IpsApi regex_api = OPT_TYPE_DETECTION, 0, 0, nullptr, - regex_pterm, + nullptr, nullptr, nullptr, regex_ctor, diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index 7bb04a323..8dad8124b 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -32,6 +32,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashfcn.h" +#include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "log/obfuscator.h" #include "main/snort_config.h" @@ -49,12 +50,7 @@ using namespace snort; #define SD_SOCIAL_NODASHES_PATTERN R"(\d{9})" #define SD_CREDIT_PATTERN_ALL R"(\d{4}\D?\d{4}\D?\d{2}\D?\d{2}\D?\d{3,4})" -// we need to update scratch in the main thread as each pattern is processed -// and then clone to thread specific after all rules are loaded. s_scratch is -// a prototype that is large enough for all uses. - -static hs_scratch_t* s_scratch = nullptr; -static unsigned scratch_index; +static HyperScratchAllocator* scratcher = nullptr; struct SdStats { @@ -133,12 +129,8 @@ private: SdPatternOption::SdPatternOption(const SdPatternConfig& c) : IpsOption(s_name, RULE_OPTION_TYPE_BUFFER_USE), config(c) { - if ( hs_error_t err = hs_alloc_scratch(config.db, &s_scratch) ) - { - // FIXIT-L why is this failing but everything is working? - ParseError("can't initialize sd_pattern for %s (%d) %p", - config.pii.c_str(), err, (void*)s_scratch); - } + if ( !scratcher->allocate(config.db) ) + ParseError("can't allocate scratch for sd_pattern '%s'", config.pii.c_str()); config.pmd.pattern_buf = config.pii.c_str(); config.pmd.pattern_size = config.pii.size(); @@ -249,13 +241,10 @@ unsigned SdPatternOption::SdSearch(const Cursor& c, Packet* p) const uint8_t* buf = c.start(); unsigned int buflen = c.length(); - std::vector ss = SnortConfig::get_conf()->state[get_instance_id()]; - assert(ss[scratch_index]); - hsContext ctx(config, p, start, buf, buflen); hs_error_t stat = hs_scan(config.db, (const char*)buf, buflen, 0, - (hs_scratch_t*)ss[scratch_index], hs_match, (void*)&ctx); + scratcher->get(), hs_match, (void*)&ctx); if ( stat == HS_SCAN_TERMINATED ) ++s_stats.terminated; @@ -300,10 +289,10 @@ class SdPatternModule : public Module { public: SdPatternModule() : Module(s_name, s_help, s_params) - { - scratch_index = SnortConfig::request_scratch( - SdPatternModule::scratch_setup, SdPatternModule::scratch_cleanup); - } + { scratcher = new HyperScratchAllocator; } + + ~SdPatternModule() override + { delete scratcher; } bool begin(const char*, int, SnortConfig*) override; bool set(const char*, Value& v, SnortConfig*) override; @@ -326,9 +315,6 @@ public: private: SdPatternConfig config; - - static void scratch_setup(SnortConfig*); - static void scratch_cleanup(SnortConfig*); }; bool SdPatternModule::begin(const char*, int, SnortConfig*) @@ -397,37 +383,6 @@ bool SdPatternModule::end(const char*, int, SnortConfig*) return true; } -//------------------------------------------------------------------------- -// public methods -//------------------------------------------------------------------------- - -void SdPatternModule::scratch_setup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - std::vector& ss = sc->state[i]; - - if ( s_scratch ) - hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss[scratch_index]); - else - ss[scratch_index] = nullptr; - } -} - -void SdPatternModule::scratch_cleanup(SnortConfig* sc) -{ - for ( unsigned i = 0; i < sc->num_slots; ++i ) - { - std::vector& ss = sc->state[i]; - - if ( ss[scratch_index] ) - { - hs_free_scratch((hs_scratch_t*)ss[scratch_index]); - ss[scratch_index] = nullptr; - } - } -} - //------------------------------------------------------------------------- // api methods //------------------------------------------------------------------------- diff --git a/src/ips_options/test/CMakeLists.txt b/src/ips_options/test/CMakeLists.txt index 9d89ffb12..7c2c4fb34 100644 --- a/src/ips_options/test/CMakeLists.txt +++ b/src/ips_options/test/CMakeLists.txt @@ -6,6 +6,8 @@ if ( HAVE_HYPERSCAN ) ../../framework/module.cc ../../framework/ips_option.cc ../../framework/value.cc + ../../helpers/scratch_allocator.cc + ../../helpers/hyper_scratch_allocator.cc ../../sfip/sf_ip.cc $ LIBS diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc index ed5e9ed6c..37fa965d2 100644 --- a/src/ips_options/test/ips_regex_test.cc +++ b/src/ips_options/test/ips_regex_test.cc @@ -51,9 +51,7 @@ SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; static std::vector s_state; - -ScScratchFunc scratch_setup; -ScScratchFunc scratch_cleanup; +static ScratchAllocator* scratcher = nullptr; SnortConfig::SnortConfig(const SnortConfig* const) { @@ -63,15 +61,15 @@ SnortConfig::SnortConfig(const SnortConfig* const) SnortConfig::~SnortConfig() = default; -int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +int SnortConfig::request_scratch(ScratchAllocator* s) { - scratch_setup = setup; - scratch_cleanup = cleanup; + scratcher = s; s_state.resize(1); - return 0; } +void SnortConfig::release_scratch(int) { } + SnortConfig* SnortConfig::get_conf() { return snort_conf; } @@ -120,9 +118,8 @@ static const Parameter* get_param(Module* m, const char* s) return nullptr; } -static IpsOption* get_option(const char* pat, bool relative = false) +static IpsOption* get_option(Module* mod, const char* pat, bool relative = false) { - Module* mod = ips_regex->mod_ctor(); mod->begin(ips_regex->name, 0, nullptr); Value vs(pat); @@ -140,7 +137,6 @@ static IpsOption* get_option(const char* pat, bool relative = false) IpsApi* api = (IpsApi*)ips_regex; IpsOption* opt = api->ctor(mod, nullptr); - ips_regex->mod_dtor(mod); return opt; } @@ -273,25 +269,28 @@ TEST(ips_regex_module, config_fail_regex) TEST_GROUP(ips_regex_option) { + Module* mod = nullptr; IpsOption* opt = nullptr; + bool do_cleanup = false; void setup() override { - opt = get_option(" foo "); - scratch_setup(snort_conf); + mod = ips_regex->mod_ctor(); + opt = get_option(mod, " foo "); } void teardown() override { IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt); - scratch_cleanup(snort_conf); - api->pterm(snort_conf); + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + ips_regex->mod_dtor(mod); } }; TEST(ips_regex_option, hash) { - IpsOption* opt2 = get_option("bar"); + IpsOption* opt2 = get_option(mod, "bar"); CHECK(opt2); CHECK(*opt != *opt2); @@ -299,23 +298,29 @@ TEST(ips_regex_option, hash) uint32_t h2 = opt2->hash(); CHECK(h1 != h2); + do_cleanup = scratcher->setup(snort_conf); + IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt2); } TEST(ips_regex_option, opeq) { - IpsOption* opt2 = get_option(" foo "); + IpsOption* opt2 = get_option(mod, " foo "); CHECK(opt2); // this is forced unequal for now CHECK(*opt != *opt2); + do_cleanup = scratcher->setup(snort_conf); + IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt2); } TEST(ips_regex_option, match_absolute) { + do_cleanup = scratcher->setup(snort_conf); + Packet pkt; pkt.data = (uint8_t*)"* foo stew *"; pkt.dsize = strlen((char*)pkt.data); @@ -328,6 +333,8 @@ TEST(ips_regex_option, match_absolute) TEST(ips_regex_option, no_match_delta) { + do_cleanup = scratcher->setup(snort_conf); + Packet pkt; pkt.data = (uint8_t*)"* foo stew *"; pkt.dsize = strlen((char*)pkt.data); @@ -344,23 +351,29 @@ TEST(ips_regex_option, no_match_delta) TEST_GROUP(ips_regex_option_relative) { + Module* mod = nullptr; IpsOption* opt = nullptr; + bool do_cleanup = false; void setup() override { - opt = get_option("\\bfoo", true); - scratch_setup(snort_conf); + mod = ips_regex->mod_ctor(); + opt = get_option(mod, "\\bfoo", true); } void teardown() override { IpsApi* api = (IpsApi*)ips_regex; api->dtor(opt); - scratch_cleanup(snort_conf); + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + ips_regex->mod_dtor(mod); } }; TEST(ips_regex_option_relative, no_match) { + do_cleanup = scratcher->setup(snort_conf); + Packet pkt; pkt.data = (uint8_t*)"* foo stew *"; pkt.dsize = strlen((char*)pkt.data); diff --git a/src/main/modules.cc b/src/main/modules.cc index 4ac3e3b14..d4f3afd22 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -81,6 +81,11 @@ static const Parameter detection_params[] = { "global_rule_state", Parameter::PT_BOOL, nullptr, "false", "apply rule_state against all policies" }, +#ifdef HAVE_HYPERSCAN + { "hyperscan_literals", Parameter::PT_BOOL, nullptr, "false", + "use hyperscan for content literal searches instead of boyer-moore" }, +#endif + { "offload_limit", Parameter::PT_INT, "0:max32", "99999", "minimum sizeof PDU to offload fast pattern search (defaults to disabled)" }, @@ -88,7 +93,7 @@ static const Parameter detection_params[] = "maximum number of simultaneous offloads (defaults to disabled)" }, { "pcre_enable", Parameter::PT_BOOL, nullptr, "true", - "disable pcre pattern matching" }, + "enable pcre pattern matching" }, { "pcre_match_limit", Parameter::PT_INT, "0:max32", "1500", "limit pcre backtracking, 0 = off" }, @@ -96,6 +101,9 @@ static const Parameter detection_params[] = { "pcre_match_limit_recursion", Parameter::PT_INT, "0:max32", "1500", "limit pcre stack consumption, 0 = off" }, + { "pcre_override", Parameter::PT_BOOL, nullptr, "true", + "enable pcre match limit overrides when pattern matching (ie ignore /O)" }, + { "enable_address_anomaly_checks", Parameter::PT_BOOL, nullptr, "false", "enable check and alerting of address anomalies" }, @@ -144,6 +152,11 @@ bool DetectionModule::set(const char* fqn, Value& v, SnortConfig* sc) else if ( v.is("global_rule_state") ) sc->global_rule_state = v.get_bool(); +#ifdef HAVE_HYPERSCAN + else if ( v.is("hyperscan_literals") ) + sc->hyperscan_literals = v.get_bool(); +#endif + else if ( v.is("offload_limit") ) sc->offload_limit = v.get_uint32(); @@ -190,6 +203,9 @@ bool DetectionModule::set(const char* fqn, Value& v, SnortConfig* sc) } } + else if ( v.is("pcre_override") ) + sc->pcre_override = v.get_bool(); + else if ( v.is("enable_address_anomaly_checks") ) sc->address_anomaly_check_enabled = v.get_bool(); diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 7df8c0e32..140d15d9e 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -85,7 +85,8 @@ SnortConfig* parser_conf = nullptr; THREAD_LOCAL SnortConfig* snort_conf = nullptr; uint32_t SnortConfig::warning_flags = 0; -static std::vector > scratch_handlers; + +static std::vector scratch_handlers; //------------------------------------------------------------------------- // private implementation @@ -251,16 +252,8 @@ SnortConfig::~SnortConfig() FreeClassifications(classifications); FreeReferences(references); - // Only call scratch cleanup if we actually called scratch setup - if ( state and state[0].size() > 0 ) - { - for ( unsigned i = scratch_handlers.size(); i > 0; i-- ) - { - if ( scratch_handlers[i - 1].second ) - scratch_handlers[i - 1].second(this); - } - // FIXIT-L: Do we need to shrink_to_fit() state->scratch at this point? - } + for ( auto* s : scratchers ) + s->cleanup(this); FreeRuleLists(this); PortTablesFree(port_tables); @@ -346,19 +339,16 @@ void SnortConfig::setup() void SnortConfig::post_setup() { - unsigned i; unsigned int handler_count = scratch_handlers.size(); // Ensure we have allocated the scratch space vector for each thread - for ( i = 0; i < num_slots; ++i ) - { + for ( unsigned i = 0; i < num_slots; ++i ) state[i].resize(handler_count); - } - for ( i = 0; i < handler_count; ++i ) + for ( auto* s : scratch_handlers ) { - if ( scratch_handlers[i].first ) - scratch_handlers[i].first(this); + if ( s and s->setup(this) ) + scratchers.push_back(s); } } @@ -1072,15 +1062,21 @@ bool SnortConfig::tunnel_bypass_enabled(uint8_t proto) return (!((get_conf()->tunnel_mask & proto) or SFDAQ::get_tunnel_bypass(proto))); } -SO_PUBLIC int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +SO_PUBLIC int SnortConfig::request_scratch(ScratchAllocator* s) { - scratch_handlers.emplace_back(std::make_pair(setup, cleanup)); + scratch_handlers.emplace_back(s); // We return an index that the caller uses to reference their per thread // scratch space return scratch_handlers.size() - 1; } +SO_PUBLIC void SnortConfig::release_scratch(int id) +{ + assert((unsigned)id < scratch_handlers.size()); + scratch_handlers[id] = nullptr; +} + SO_PUBLIC SnortConfig* SnortConfig::get_parser_conf() { return parser_conf; } diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 9347f9f5d..a92e76b35 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -30,6 +30,7 @@ #include "events/event_queue.h" #include "framework/bits.h" +#include "helpers/scratch_allocator.h" #include "main/policy.h" #include "main/thread.h" #include "sfip/sf_cidr.h" @@ -152,8 +153,6 @@ struct GHash; struct XHash; struct SnortConfig; -typedef void (* ScScratchFunc)(SnortConfig* sc); - class ReloadResourceTuner { public: @@ -233,7 +232,9 @@ public: // somehow a packet thread needs a much lower setting long int pcre_match_limit = 1500; long int pcre_match_limit_recursion = 1500; + int pcre_ovector_size = 0; + bool pcre_override = true; int asn1_mem = 0; uint32_t run_flags = 0; @@ -241,6 +242,10 @@ public: unsigned offload_limit = 99999; // disabled unsigned offload_threads = 0; // disabled +#ifdef HAVE_HYPERSCAN + bool hyperscan_literals = false; +#endif + bool global_rule_state = false; bool global_default_rule_state = true; @@ -396,6 +401,7 @@ public: MemoryConfig* memory = nullptr; //------------------------------------------------------ + std::vector scratchers; std::vector* state = nullptr; unsigned num_slots = 0; @@ -710,7 +716,8 @@ public: // This requests an entry in the scratch space vector and calls setup / // cleanup as appropriate - SO_PUBLIC static int request_scratch(ScScratchFunc setup, ScScratchFunc cleanup); + SO_PUBLIC static int request_scratch(ScratchAllocator*); + SO_PUBLIC static void release_scratch(int); // Use this to access current thread's conf from other units static void set_conf(SnortConfig*); diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc index 63dc36c58..a7a228213 100644 --- a/src/search_engines/hyperscan.cc +++ b/src/search_engines/hyperscan.cc @@ -28,7 +28,9 @@ #include #include +#include "framework/module.h" #include "framework/mpse.h" +#include "helpers/scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" #include "main/thread.h" @@ -36,6 +38,9 @@ using namespace snort; +static const char* s_name = "hyperscan"; +static const char* s_help = "intel hyperscan-based mpse with regex support"; + struct Pattern { std::string pat; @@ -101,6 +106,19 @@ typedef std::vector PatternVector; static std::vector s_scratch; static unsigned int scratch_index; static bool scratch_registered = false; +static ScratchAllocator* scratcher = nullptr; + +struct ScanContext +{ + class HyperscanMpse* mpse; + MpseMatch match_cb; + void* match_ctx; + int nfound = 0; + + ScanContext(HyperscanMpse* m, MpseMatch cb, void* ctx) + { mpse = m; match_cb = cb; match_ctx = ctx; } + +}; //------------------------------------------------------------------------- // mpse @@ -142,7 +160,7 @@ public: int get_pattern_count() const override { return pvector.size(); } - int match(unsigned id, unsigned long long to); + int match(unsigned id, unsigned long long to, MpseMatch match_cb, void* match_ctx); static int match( unsigned id, unsigned long long from, unsigned long long to, @@ -157,19 +175,11 @@ private: hs_database_t* hs_db = nullptr; - static THREAD_LOCAL MpseMatch match_cb; - static THREAD_LOCAL void* match_ctx; - static THREAD_LOCAL int nfound; - public: static uint64_t instances; static uint64_t patterns; }; -THREAD_LOCAL MpseMatch HyperscanMpse::match_cb = nullptr; -THREAD_LOCAL void* HyperscanMpse::match_ctx = nullptr; -THREAD_LOCAL int HyperscanMpse::nfound = 0; - uint64_t HyperscanMpse::instances = 0; uint64_t HyperscanMpse::patterns = 0; @@ -256,11 +266,10 @@ int HyperscanMpse::prep_patterns(SnortConfig* sc) return 0; } -int HyperscanMpse::match(unsigned id, unsigned long long to) +int HyperscanMpse::match(unsigned id, unsigned long long to, MpseMatch match_cb, void* match_ctx) { assert(id < pvector.size()); Pattern& p = pvector[id]; - nfound++; return match_cb(p.user, p.user_tree, (int)to, match_ctx, p.user_list); } @@ -268,18 +277,16 @@ int HyperscanMpse::match( unsigned id, unsigned long long /*from*/, unsigned long long to, unsigned /*flags*/, void* pv) { - HyperscanMpse* h = (HyperscanMpse*)pv; - return h->match(id, to); + ScanContext* scan = (ScanContext*)pv; + scan->nfound++; + return scan->mpse->match(id, to, scan->match_cb, scan->match_ctx); } int HyperscanMpse::_search( const uint8_t* buf, int n, MpseMatch mf, void* pv, int* current_state) { *current_state = 0; - nfound = 0; - - match_cb = mf; - match_ctx = pv; + ScanContext scan(this, mf, pv); hs_scratch_t* ss = (hs_scratch_t*)SnortConfig::get_conf()->state[get_instance_id()][scratch_index]; @@ -287,17 +294,19 @@ int HyperscanMpse::_search( // scratch is null for the degenerate case w/o patterns assert(!hs_db or ss); - hs_scan(hs_db, (const char*)buf, n, 0, ss, - HyperscanMpse::match, this); + hs_scan(hs_db, (const char*)buf, n, 0, ss, HyperscanMpse::match, &scan); - return nfound; + return scan.nfound; } -static void scratch_setup(SnortConfig* sc) +static bool scratch_setup(SnortConfig* sc) { // find the largest scratch and clone for all slots hs_scratch_t* max = nullptr; + if ( !s_scratch.size() ) + return false; + for ( unsigned i = 0; i < sc->num_slots; ++i ) { if ( !s_scratch[i] ) @@ -324,44 +333,58 @@ static void scratch_setup(SnortConfig* sc) } s_scratch[i] = nullptr; } + if ( !max ) + return false; + for ( unsigned i = 0; i < sc->num_slots; ++i ) { hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index]; - - if ( max ) - hs_clone_scratch(max, ss); - else - *ss = nullptr; + hs_clone_scratch(max, ss); } - if ( max ) - hs_free_scratch(max); + hs_free_scratch(max); + return true; } static void scratch_cleanup(SnortConfig* sc) { for ( unsigned i = 0; i < sc->num_slots; ++i ) { - hs_scratch_t* ss = (hs_scratch_t*) sc->state[i][scratch_index]; - - if ( ss ) - { - hs_free_scratch(ss); - sc->state[i][scratch_index] = nullptr; - } + hs_scratch_t* ss = (hs_scratch_t*)sc->state[i][scratch_index]; + hs_free_scratch(ss); + sc->state[i][scratch_index] = nullptr; } } +class HyperscanModule : public Module +{ +public: + HyperscanModule() : Module(s_name, s_help) + { + scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup); + scratch_index = scratcher->get_id(); + } + + ~HyperscanModule() override + { delete scratcher; } +}; + //------------------------------------------------------------------------- // api //------------------------------------------------------------------------- +static Module* mod_ctor() +{ return new HyperscanModule; } + +static void mod_dtor(Module* p) +{ delete p; } + static Mpse* hs_ctor( SnortConfig* sc, class Module*, const MpseAgent* a) { if ( !scratch_registered ) { s_scratch.resize(sc->num_slots); - scratch_index = SnortConfig::request_scratch(scratch_setup, scratch_cleanup); + scratch_index = scratcher->get_id(); scratch_registered = true; } return new HyperscanMpse(sc, a); @@ -393,10 +416,10 @@ static const MpseApi hs_api = 0, API_RESERVED, API_OPTIONS, - "hyperscan", - "intel hyperscan-based mpse with regex support", - nullptr, - nullptr + s_name, + s_help, + mod_ctor, + mod_dtor }, MPSE_REGEX | MPSE_MTBLD, nullptr, // activate diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc index 49f1fb625..49a360e4e 100644 --- a/src/search_engines/search_tool.cc +++ b/src/search_engines/search_tool.cc @@ -130,8 +130,7 @@ int SearchTool::find( num = mpsegrp->get_offload_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); if ( num < 0 ) - num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, - &state); + num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); } else num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); diff --git a/src/search_engines/test/CMakeLists.txt b/src/search_engines/test/CMakeLists.txt index 82f81ff30..6769b4cb5 100644 --- a/src/search_engines/test/CMakeLists.txt +++ b/src/search_engines/test/CMakeLists.txt @@ -10,7 +10,11 @@ add_cpputest( search_tool_test if ( HAVE_HYPERSCAN ) add_cpputest( hyperscan_test - SOURCES ../hyperscan.cc + SOURCES + ../hyperscan.cc + ../../framework/module.cc + ../../helpers/scratch_allocator.cc + ../../helpers/hyper_scratch_allocator.cc LIBS ${HS_LIBRARIES} ) endif() diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index 2a0245248..80605218c 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -25,9 +25,11 @@ #include #include "framework/base_api.h" +#include "framework/counts.h" #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "main/snort_config.h" +#include "utils/stats.h" // must appear after snort_config.h to avoid broken c++ map include #include @@ -38,6 +40,7 @@ using namespace snort; //------------------------------------------------------------------------- // base stuff //------------------------------------------------------------------------- + namespace snort { Mpse::Mpse(const char*) { } @@ -96,9 +99,7 @@ SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; static std::vector s_state; - -ScScratchFunc scratch_setup; -ScScratchFunc scratch_cleanup; +static ScratchAllocator* scratcher = nullptr; SnortConfig::SnortConfig(const SnortConfig* const) { @@ -108,15 +109,15 @@ SnortConfig::SnortConfig(const SnortConfig* const) SnortConfig::~SnortConfig() = default; -int SnortConfig::request_scratch(ScScratchFunc setup, ScScratchFunc cleanup) +int SnortConfig::request_scratch(ScratchAllocator* s) { - scratch_setup = setup; - scratch_cleanup = cleanup; + scratcher = s; s_state.resize(1); - return 0; } +void SnortConfig::release_scratch(int) { } + SnortConfig* SnortConfig::get_conf() { return snort_conf; } @@ -131,6 +132,9 @@ unsigned get_instance_id() { return 0; } } +void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } +void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } + //------------------------------------------------------------------------- // stubs, spies, etc. //------------------------------------------------------------------------- @@ -212,7 +216,9 @@ TEST(mpse_hs_base, mpse) TEST_GROUP(mpse_hs_match) { + Module* mod = nullptr; Mpse* hs = nullptr; + bool do_cleanup = false; const MpseApi* mpse_api = (MpseApi*)se_hyperscan; void setup() override @@ -220,6 +226,7 @@ TEST_GROUP(mpse_hs_match) // FIXIT-L cpputest hangs or crashes in the leak detector MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); CHECK(se_hyperscan); + mod = mpse_api->base.mod_ctor(); hs = mpse_api->ctor(snort_conf, nullptr, &s_agent); CHECK(hs); hits = 0; @@ -228,7 +235,9 @@ TEST_GROUP(mpse_hs_match) void teardown() override { mpse_api->dtor(hs); - scratch_cleanup(snort_conf); + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + mpse_api->base.mod_dtor(mod); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; @@ -239,6 +248,8 @@ TEST(mpse_hs_match, empty) CHECK(parse_errors == 0); CHECK(hs->get_pattern_count() == 0); + do_cleanup = scratcher->setup(snort_conf); + int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 0); CHECK(hits == 0); @@ -252,7 +263,7 @@ TEST(mpse_hs_match, single) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - scratch_setup(snort_conf); + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -267,7 +278,7 @@ TEST(mpse_hs_match, nocase) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - scratch_setup(snort_conf); + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -283,7 +294,7 @@ TEST(mpse_hs_match, other) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - scratch_setup(snort_conf); + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -301,7 +312,8 @@ TEST(mpse_hs_match, multi) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 3); - scratch_setup(snort_conf); + + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 3); @@ -318,7 +330,8 @@ TEST(mpse_hs_match, regex) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - scratch_setup(snort_conf); + + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 0); @@ -335,7 +348,8 @@ TEST(mpse_hs_match, pcre) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - scratch_setup(snort_conf); + + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((uint8_t*)":definition(", 12, match, nullptr, &state) == 0); @@ -352,8 +366,10 @@ TEST(mpse_hs_match, pcre) TEST_GROUP(mpse_hs_multi) { + Module* mod = nullptr; Mpse* hs1 = nullptr; Mpse* hs2 = nullptr; + bool do_cleanup = false; const MpseApi* mpse_api = (MpseApi*)se_hyperscan; void setup() override @@ -362,6 +378,8 @@ TEST_GROUP(mpse_hs_multi) MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); CHECK(se_hyperscan); + mod = mpse_api->base.mod_ctor(); + hs1 = mpse_api->ctor(snort_conf, nullptr, &s_agent); CHECK(hs1); @@ -375,7 +393,9 @@ TEST_GROUP(mpse_hs_multi) { mpse_api->dtor(hs1); mpse_api->dtor(hs2); - scratch_cleanup(snort_conf); + if ( do_cleanup ) + scratcher->cleanup(snort_conf); + mpse_api->base.mod_dtor(mod); MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; @@ -393,7 +413,7 @@ TEST(mpse_hs_multi, single) CHECK(hs1->get_pattern_count() == 1); CHECK(hs2->get_pattern_count() == 1); - scratch_setup(snort_conf); + do_cleanup = scratcher->setup(snort_conf); int state = 0; CHECK(hs1->search((uint8_t*)"fubar", 5, match, nullptr, &state) == 1 ); diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 4bbe141a2..050445200 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -24,24 +24,24 @@ set( UTIL_INCLUDES add_library ( utils OBJECT ${UTIL_INCLUDES} ${SNPRINTF_SOURCES} - boyer_moore.cc + boyer_moore.cc dnet_header.h dyn_array.cc dyn_array.h kmap.cc - segment_mem.cc - sflsq.cc - sfmemcap.cc + segment_mem.cc + sflsq.cc + sfmemcap.cc snort_bounds.h stats.cc util.cc util_ber.cc util_cstring.cc - util_jsnorm.cc - util_net.cc + util_jsnorm.cc + util_net.cc util_net.h - util_unfold.cc - util_utf.cc + util_unfold.cc + util_utf.cc ${TEST_FILES} )