ips_rem.cc
ips_rev.cc
ips_rpc.cc
- ips_sd_pattern.cc
- sd_credit_card.cc
- sd_credit_card.h
- sd_pattern_match.cc
- sd_pattern_match.h
ips_seq.cc
ips_session.cc
ips_sid.cc
)
if ( HAVE_HYPERSCAN )
- set(OPTION_LIST ips_regex.cc ips_regex.h)
+ set(OPTION_LIST ips_regex.cc ips_regex.h
+ ips_sd_pattern.cc ips_sd_pattern.h
+ sd_credit_card.cc sd_credit_card.h)
endif ()
if (STATIC_IPS_OPTIONS)
add_shared_library(ips_rem ips_options ips_rem.cc)
add_shared_library(ips_rev ips_options ips_rev.cc)
add_shared_library(ips_rpc ips_options ips_rpc.cc)
- add_shared_library(ips_sd_pattern ips_options ips_sd_pattern.cc sd_credit_card.cc sd_credit_card.h sd_pattern_match.cc sd_pattern_match.h)
add_shared_library(ips_sid ips_options ips_sid.cc)
add_shared_library(ips_seq ips_options ips_seq.cc)
add_shared_library(ips_session ips_options ips_session.cc)
ips_rem.cc \
ips_rev.cc \
ips_rpc.cc \
-ips_sd_pattern.cc \
-sd_credit_card.cc sd_credit_card.h \
-sd_pattern_match.cc sd_pattern_match.h \
ips_seq.cc \
ips_session.cc \
ips_sid.cc \
if HAVE_HYPERSCAN
libips_options_a_SOURCES += ips_regex.cc ips_regex.h
+libips_options_a_SOURCES += ips_sd_pattern.cc ips_sd_pattern.h sd_credit_card.cc sd_credit_card.h
endif
if STATIC_IPS_OPTIONS
libips_rpc_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared
libips_rpc_la_SOURCES = ips_rpc.cc
-optlib_LTLIBRARIES += libips_sd_pattern.la
-libips_sd_pattern_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO
-libips_sd_pattern_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared
-libips_sd_pattern_la_SOURCES = ips_sd_pattern.cc sd_credit_card.cc \
- sd_credit_card.h sd_pattern_match.cc sd_pattern_match.h
-
optlib_LTLIBRARIES += libips_seq.la
libips_seq_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO
libips_seq_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared
semantics. The Snort 2X options had various implementations of ranges so
3X differs in some places.
+The "regex" and "sd_pattern" options both use hyperscan for pattern matching.
+Hyperscan is an "optional" dependency for Snort3; These rule options will
+not exist without satisfying that dependency.
+
+Hyperscan documentation can be found online
+http://01org.github.io/hyperscan/dev-reference
+
+The "sd_pattern" will be used as a fast pattern in the future (like "regex")
+for performance.
// ips_sd_pattern.cc author Victor Roemer <viroemer@cisco.com>
-// FIXIT-M use Hyperscan
+#include "ips_sd_pattern.h"
#include <string.h>
#include <assert.h>
#include <string>
+#include <hs_compile.h>
+#include <hs_runtime.h>
+
#include "framework/cursor.h"
#include "framework/ips_option.h"
#include "framework/module.h"
#include "main/thread.h"
#include "parser/parser.h"
#include "profiler/profiler.h"
-#include "sd_pattern_match.h"
+#include "sd_credit_card.h"
#include "log/obfuscator.h"
#define s_name "sd_pattern"
#define s_help "rule option for detecting sensitive data"
+#define SD_SOCIAL_PATTERN "\\b\\d{3}-\\d{2}-\\d{4}\\b"
+#define SD_SOCIAL_NODASHES_PATTERN "\\b\\d{9}\\b"
+#define SD_CREDIT_PATTERN_ALL "\\b\\d{4}[- ]?\\d{4}[- ]?\\d{2}[- ]?\\d{2}[- ]?\\d{3,4}\\b"
+
+// we need to update scratch in the main thread as each pattern is processed
+// and then clone to thread specific after all rules are loaded. s_scratch is
+// a prototype that is large enough for all uses.
+
+// FIXIT-L Determine if it's worthwhile to use a single scratch space for both
+// "regex" and "sd_pattern" keywords.
+// FIXIT-L See ips_regex.cc for more information.
+static hs_scratch_t* s_scratch = nullptr;
+
struct SdStats
{
- PegCount nomatch_notfound;
PegCount nomatch_threshold;
+ PegCount nomatch_notfound;
+ PegCount terminated;
};
const PegInfo sd_pegs[] =
{
{ "below threshold", "sd_pattern matched but missed threshold" },
{ "pattern not found", "sd_pattern did not not match" },
+ { "terminated", "hyperscan terminated" },
{ nullptr, nullptr }
};
struct SdPatternConfig
{
+ hs_database_t* db;
+
std::string pii;
unsigned threshold = 1;
- bool obfuscate_pii;
+ bool obfuscate_pii = false;
+ int (*validate)(const uint8_t* buf, unsigned long long buflen) = nullptr;
+
+ inline bool operator==(const SdPatternConfig& rhs) const
+ {
+ if ( pii == rhs.pii and threshold == rhs.threshold )
+ return true;
+ return false;
+ }
};
static THREAD_LOCAL ProfileStats sd_pattern_perf_stats;
private:
unsigned SdSearch(Cursor&, Packet*);
-
const SdPatternConfig config;
- SdOptionData* opt;
};
SdPatternOption::SdPatternOption(const SdPatternConfig& c) :
IpsOption(s_name, RULE_OPTION_TYPE_BUFFER_USE), config(c)
{
- opt = new SdOptionData(config.pii, config.obfuscate_pii);
+ if ( hs_error_t err = hs_alloc_scratch(config.db, &s_scratch) )
+ {
+ // FIXIT-L why is this failing but everything is working?
+ ParseError("can't initialize sd_pattern for %s (%d) %p",
+ config.pii.c_str(), err, (void*)s_scratch);
+ }
}
SdPatternOption::~SdPatternOption()
-{
- delete opt;
+{
+ if ( config.db )
+ hs_free_database(config.db);
}
uint32_t SdPatternOption::hash() const
{
- uint32_t a = 0, b = 0, c = 0;
+ uint32_t a = 0, b = 0, c = config.threshold;
mix_str(a, b, c, config.pii.c_str());
mix_str(a, b, c, get_name());
finalize(a, b, c);
const SdPatternOption& rhs = static_cast<const SdPatternOption&>(ips);
- if ( config.pii == rhs.config.pii
- and config.threshold == rhs.config.threshold )
+ if ( config == rhs.config )
return true;
return false;
}
+struct hsContext
+{
+ hsContext(const SdPatternConfig &c_, Packet* p_, const uint8_t* const start_)
+ : config(c_), packet(p_), start(start_) {}
+
+ unsigned int count = 0;
+
+ SdPatternConfig config;
+ Packet* packet = nullptr;
+ const uint8_t* const start = nullptr;
+ const uint8_t* buf = nullptr;
+};
+
+// FIXIT-H Count matches
+// FIXIT-H afix this to SdPatternOption
+int hs_match(unsigned int /*id*/, unsigned long long from,
+ unsigned long long to, unsigned int /*flags*/, void *context)
+{
+ hsContext* ctx = (hsContext*) context;
+
+ assert(ctx);
+ assert(ctx->packet);
+ assert(ctx->start);
+
+ unsigned long long len = to - from;
+ if ( ctx->config.validate && ctx->config.validate(ctx->buf, len) != 1 )
+ return 0;
+
+ ctx->count++;
+
+ if ( ctx->config.obfuscate_pii )
+ {
+ if ( !ctx->packet->obfuscator )
+ ctx->packet->obfuscator = new Obfuscator();
+
+ uint32_t off = ctx->buf - ctx->start;
+ // FIXIT-L Make configurable or don't show any PII partials (0 for user defined??)
+ len = len > 4 ? len - 4 : len;
+ ctx->packet->obfuscator->push(off, len);
+ }
+
+ return 0;
+}
+
unsigned SdPatternOption::SdSearch(Cursor& c, Packet* p)
{
const uint8_t* const start = c.buffer();
const uint8_t* buf = c.start();
- uint16_t buflen = c.length();
- const uint8_t* const end = buf + buflen;
+ unsigned int buflen = c.length();
- unsigned count = 0;
- while (buf < end && count < config.threshold)
- {
- uint16_t match_len = 0;
+ SnortState* ss = snort_conf->state + get_instance_id();
+ assert(ss->sdpattern_scratch);
- if ( opt->match(buf, &match_len, buflen) )
- {
- if ( opt->obfuscate_pii )
- {
- if ( !p->obfuscator )
- p->obfuscator = new Obfuscator();
-
- uint32_t off = buf - start;
- p->obfuscator->push(off, match_len - 4);
- }
-
- buf += match_len;
- buflen -= match_len;
- count++;
- }
- else
- {
- buf++;
- buflen--;
- }
- }
+ hsContext ctx(config, p, start);
+ ctx.buf = buf;
+
+ hs_error_t stat = hs_scan(config.db, (const char*)buf, buflen, 0,
+ (hs_scratch_t*)ss->sdpattern_scratch, hs_match, (void*)&ctx);
- return count;
+ if ( stat == HS_SCAN_TERMINATED )
+ ++s_stats.terminated;
+
+ return ctx.count;
}
int SdPatternOption::eval(Cursor& c, Packet* p)
bool begin(const char*, int, SnortConfig*) override;
bool set(const char*, Value& v, SnortConfig*) override;
+ bool end(const char*, int, SnortConfig*) override;
const PegInfo* get_pegs() const override
{ return sd_pegs; }
bool SdPatternModule::set(const char*, Value& v, SnortConfig* sc)
{
- config.obfuscate_pii = sc->obfuscate_pii;
+ config.obfuscate_pii = false;
+
if ( v.is("~pattern") )
{
config.pii = v.get_string();
else
return false;
+ // Check if built-in pattern should be used.
+ if (config.pii == "credit_card")
+ {
+ config.pii = SD_CREDIT_PATTERN_ALL;
+ config.validate = SdLuhnAlgorithm;
+ config.obfuscate_pii = sc->obfuscate_pii;
+ }
+
+ else if (config.pii == "us_social")
+ {
+ config.pii = SD_SOCIAL_PATTERN;
+ config.obfuscate_pii = sc->obfuscate_pii;
+ }
+
+ else if (config.pii == "us_social_nodashes")
+ {
+ config.pii = SD_SOCIAL_NODASHES_PATTERN;
+ config.obfuscate_pii = sc->obfuscate_pii;
+ }
+
+ return true;
+}
+
+bool SdPatternModule::end(const char*, int, SnortConfig*)
+{
+ hs_compile_error_t* err = nullptr;
+
+ if ( hs_compile(config.pii.c_str(), HS_FLAG_DOTALL|HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, nullptr, &config.db, &err)
+ or !config.db )
+ {
+ ParseError("can't compile regex '%s'", config.pii.c_str());
+ hs_free_compile_error(err);
+ return false;
+ }
return true;
}
+//-------------------------------------------------------------------------
+// public methods
+//-------------------------------------------------------------------------
+
+void sdpattern_setup(SnortConfig* sc)
+{
+ for ( unsigned i = 0; i < sc->num_slots; ++i )
+ {
+ SnortState* ss = sc->state + i;
+
+ if ( s_scratch )
+ hs_clone_scratch(s_scratch, (hs_scratch_t**)&ss->sdpattern_scratch);
+ else
+ ss->sdpattern_scratch = nullptr;
+ }
+}
+
+void sdpattern_cleanup(SnortConfig* sc)
+{
+ for ( unsigned i = 0; i < sc->num_slots; ++i )
+ {
+ SnortState* ss = sc->state + i;
+
+ if ( ss->sdpattern_scratch )
+ {
+ hs_free_scratch((hs_scratch_t*)ss->sdpattern_scratch);
+ ss->sdpattern_scratch = nullptr;
+ }
+ }
+}
+
//-------------------------------------------------------------------------
// api methods
//-------------------------------------------------------------------------
}
static void sd_pattern_dtor(IpsOption* p)
-{ delete p; }
+{
+ delete p;
+}
static const IpsApi sd_pattern_api =
{
//--------------------------------------------------------------------------
-// Copyright (C) 2014-2016 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2009-2013 Sourcefire, Inc.
+// Copyright (C) 2015-2016 Cisco and/or its affiliates. All rights reserved.
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License Version 2 as published
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// sd_pattern_match.h author Ryan Jordan
+#ifndef IPS_SD_PATTERN_H
+#define IPS_SD_PATTERN_H
-#ifndef SD_PATTERN_MATCH_H
-#define SD_PATTERN_MATCH_H
-
-#include <iostream>
-#include <stdint.h>
-#include "utils/util.h"
-
-#define SD_SOCIAL_PATTERN "\\b\\d{3}-\\d{2}-\\d{4}\\b"
-#define SD_SOCIAL_NODASHES_PATTERN "\\b\\d{9}\\b"
-#define SD_CREDIT_PATTERN_ALL "\\b\\d{4} ?-?\\d{4} ?-?\\d{2} ?-?\\d{2} ?-?\\d{3}\\d?\\b"
-
-class SdOptionData
-{
-public:
- friend class SdPatternOption;
-
- SdOptionData(std::string pattern, bool obfuscate);
-
- ~SdOptionData()
- { snort_free(pattern); }
-
- void ExpandBrackets();
- bool match(const uint8_t* const buf, uint16_t* const buf_index, uint16_t buflen);
-
-private:
- char* pattern;
- int (*validate)(const uint8_t* buf, uint32_t buflen) = nullptr;
- bool obfuscate_pii = false;
-};
+struct SnortConfig;
+void sdpattern_setup(SnortConfig*);
+void sdpattern_cleanup(SnortConfig*);
#endif
-
*
* Returns: 1 on match, 0 otherwise.
*/
-int SdLuhnAlgorithm(const uint8_t *buf, uint32_t buflen)
+int SdLuhnAlgorithm(const uint8_t *buf, unsigned long long buflen)
{
int i, digits, alternate, sum, val;
char cc_digits[CC_COPY_BUF_LEN]; /* Normalized CC# string */
#include <stdint.h>
-int SdLuhnAlgorithm(const uint8_t *buf, uint32_t buflen);
+int SdLuhnAlgorithm(const uint8_t *buf, unsigned long long buflen);
#endif
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2016 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2009-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// sd_pattern_match.cc author Ryan Jordan
-
-#include "sd_pattern_match.h"
-#include "sd_credit_card.h"
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ctype.h>
-
-#include "log/messages.h"
-#include "utils/util.h"
-
-SdOptionData::SdOptionData(std::string pattern_, bool obfuscate_)
-{
- if (pattern_ == "credit_card")
- {
- pattern_ = SD_CREDIT_PATTERN_ALL;
- validate = SdLuhnAlgorithm;
- obfuscate_pii = obfuscate_;
- }
-
- else if (pattern_ == "us_social")
- {
- pattern_ = SD_SOCIAL_PATTERN;
- obfuscate_pii = obfuscate_;
- }
-
- else if (pattern_ == "us_social_nodashes")
- {
- pattern_ = SD_SOCIAL_NODASHES_PATTERN;
- obfuscate_pii = obfuscate_;
- }
-
- pattern = snort_strdup(pattern_.c_str());
- ExpandBrackets();
-}
-
-void SdOptionData::ExpandBrackets()
-{
- char* bracket_index, * new_pii, * endptr, * pii_position;
- unsigned long int new_pii_size, repetitions, total_reps = 0;
- unsigned int num_brackets = 0;
-
- if ( !pattern )
- return;
-
- bracket_index = strchr(pattern, '{');
-
- if ( bracket_index == pattern )
- ParseError("sd_pattern \"%s\" starts with curly brackets which have nothing to modify.", pattern);
-
- while ( bracket_index )
- {
- if ( (bracket_index > pattern) && (*(bracket_index-1) == '\\') )
- {
- // Ignore escaped brackets
- bracket_index = strchr(bracket_index+1, '{');
- continue;
- }
-
- // Check for the case of one bracket set modifying another, i.e. "{3}{4}"
- // Note: "\}{4}" is OK
- if ( (bracket_index > pattern + 1) && (*(bracket_index - 1) == '}') && (*(bracket_index - 2) != '\\') )
- ParseError("sd_pattern \"%s\" contains curly brackets which have nothing to modify.", pattern);
-
- repetitions = strtoul(bracket_index+1, &endptr, 10);
-
- if ( *endptr != '}' && *endptr != '\0' )
- ParseError("sd_pattern \"%s\" contains curly brackets with non-digits inside.", pattern);
-
- else if (*endptr == '\0')
- ParseError("sd_pattern \"%s\" contains an unterminated curly bracket.", pattern);
-
- if ( (bracket_index > pattern+1) && (*(bracket_index-2) == '\\') )
- total_reps += (repetitions * 2);
- else
- total_reps += repetitions;
-
- num_brackets++;
-
- bracket_index = strchr(bracket_index+1, '{');
- }
-
- if ( num_brackets == 0 )
- return;
-
- new_pii_size = (strlen(pattern) + total_reps - 2 * num_brackets + 1);
- new_pii = (char*)snort_calloc(new_pii_size, sizeof(*new_pii));
-
- pii_position = pattern;
-
- while (*pii_position != '\0')
- {
- char repeated_section[3] = {'\0'};
- unsigned long int i, reps = 1;
-
- repeated_section[0] = pii_position[0];
- pii_position++;
-
- if ( repeated_section[0] == '\\'
- && pii_position[0] != '\0' )
- {
- repeated_section[1] = pii_position[0];
- pii_position++;
- }
-
- if ( pii_position[0] == '{' )
- {
- reps = strtoul(pii_position+1, &endptr, 10);
- pii_position = endptr+1;
- }
-
- for (i = 0; i < reps; i++)
- strncat(new_pii, repeated_section, 2);
- }
-
- snort_free(pattern);
- pattern = new_pii;
-}
-
-bool SdOptionData::match(const uint8_t * const buf, uint16_t * const buf_index, uint16_t buflen)
-{
- uint16_t pattern_index = 0;
- bool node_match = true;
-
- while ( *buf_index < buflen && pattern[pattern_index] != '\0' && node_match )
- {
- char const * const pc = &pattern[pattern_index];
-
- if ( pc[0] == '\\' && pc[1] != '\0' )
- {
-match__rescan:
- pattern_index++;
- switch ( pattern[pattern_index] )
- {
- // Escaped special character
- case '\\':
- case '{':
- case '}':
- case '?':
- node_match = (buf[*buf_index] == pattern[pattern_index]);
- break;
-
- // \d : match digit
- case 'd':
- node_match = isdigit((int)buf[*buf_index]);
- break;
-
- // \D : match non-digit
- case 'D':
- node_match = !isdigit((int)buf[*buf_index]);
- break;
-
- // \w : match alphanumeric
- case 'w':
- node_match = isalnum((int)buf[*buf_index]);
- break;
-
- // \W : match non-alphanumeric */
- case 'W':
- node_match = !isalnum((int)buf[*buf_index]);
- break;
-
- // \l : match a letter
- case 'l':
- node_match = isalpha((int)buf[*buf_index]);
- break;
-
- // \L : match a non-letter
- case 'L':
- node_match = !isalpha((int)buf[*buf_index]);
- break;
-
- // \b : match a numeric boundary
- case 'b':
- node_match = !isdigit((int)buf[*buf_index]);
- if ( !node_match && *buf_index == 0
- && pattern[pattern_index+1] != '\0'
- && pattern[pattern_index+2] != '\0' )
- {
- pattern_index++;
- goto match__rescan;
- }
- }
- }
- else
- {
- // Normal byte match
- node_match = (buf[*buf_index] == pattern[pattern_index]);
- }
-
- // Handle optional characters
- if (pattern[pattern_index + 1] == '?')
- {
- pattern_index += 2;
- if (node_match)
- (*buf_index)++;
- else
- node_match = true;
- }
- else
- {
- (*buf_index)++;
- pattern_index++;
- }
- }
-
- if ( !node_match )
- return false;
-
- if( *buf_index == buflen )
- {
- char const * const pc = &pattern[pattern_index];
-
- // '\b' can match EOM
- if ( !(pc[0] == '\\' && pc[1] == 'b') )
- {
- if( (pc[0] == '\0') )
- return true;
-
- else
- return false;
- }
- }
-
- if ( validate && validate(buf, *buf_index) != 1 )
- return false;
-
- // Success!
- return true;
-}
-
#ifdef HAVE_HYPERSCAN
#include "ips_options/ips_regex.h"
+#include "ips_options/ips_sd_pattern.h"
#include "search_engines/hyperscan.h"
#endif
#ifdef HAVE_HYPERSCAN
hyperscan_cleanup(this);
+ sdpattern_cleanup(this);
regex_cleanup(this);
#endif
pcre_cleanup(this);
pcre_setup(this);
#ifdef HAVE_HYPERSCAN
regex_setup(this);
+ sdpattern_setup(this);
hyperscan_setup(this);
#endif
}
{
int* pcre_ovector;
- // regex and hs are conditionally built but these are unconditional to
- // avoid compatibility issues with plugins. if these are conditional
- // then API_OPTIONS must be updated. note: fwd decls don't work here.
+ // regex hyperscan and sdpattern are conditionally built but these are
+ // unconditional to avoid compatibility issues with plugins. if these are
+ // conditional then API_OPTIONS must be updated.
+ // note: fwd decls don't work here.
void* regex_scratch;
void* hyperscan_scratch;
+ void* sdpattern_scratch;
};
struct SnortConfig
projects / thirdparty / snort3.git / commitdiff
