From: Michael Altizer (mialtize) Date: Tue, 13 Sep 2016 18:38:53 +0000 (-0400) Subject: Merge pull request #618 in SNORT/snort3 from dce_fast_pattern to master X-Git-Tag: 3.0.0-233~263^2~16 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=8b1c55368414efb4bd94987cdc3904fa3aebdf2c;p=thirdparty%2Fsnort3.git Merge pull request #618 in SNORT/snort3 from dce_fast_pattern to master Squashed commit of the following: commit 1be83a6aa6114539d25f00941bce219b2d5bafab Author: mdagon Date: Tue Sep 13 12:18:04 2016 -0400 Code review commit 30ad3f627238c4917920b1e4699331e391235edb Author: mdagon Date: Mon Sep 12 03:01:28 2016 -0400 Dce iface fast pattern for tcp --- diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index 9a027914c..33ce1c633 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -310,13 +310,14 @@ void print_option_tree(detection_option_tree_node_t* node, int level) DEBUG_WRAP( DebugFormat(DEBUG_DETECT, "%d%*s%*d 0x%x\n", - level, indent - offset, option_type_str[node->option_type], - 54 - indent, node->num_children, node->option_data); + level, indent - offset, option_type_str[node->option_type], + 54 - indent, node->num_children, node->option_data); for (i=0; inum_children; i++) print_option_tree(node->children[i], level+1); ); } + #endif void* add_detection_option_tree( @@ -370,13 +371,13 @@ int detection_option_node_evaluate( auto last_check = state.last_check; if ( last_check.ts == p->pkth->ts && - last_check.packet_number == cur_eval_pkt_count && - last_check.rebuild_flag == (p->packet_flags & PKT_REBUILT_STREAM) && - !(p->packet_flags & PKT_ALLOW_MULTIPLE_DETECT) ) + last_check.packet_number == cur_eval_pkt_count && + last_check.rebuild_flag == (p->packet_flags & PKT_REBUILT_STREAM) && + !(p->packet_flags & PKT_ALLOW_MULTIPLE_DETECT) ) { if ( !last_check.flowbit_failed && - !(p->packet_flags & PKT_IP_RULE_2ND) && - !(p->proto_bits & (PROTO_BIT__TEREDO|PROTO_BIT__GTP)) ) + !(p->packet_flags & PKT_IP_RULE_2ND) && + !(p->proto_bits & (PROTO_BIT__TEREDO|PROTO_BIT__GTP)) ) { return last_check.result; } @@ -397,7 +398,7 @@ int detection_option_node_evaluate( IpsOption* opt = (IpsOption*)node->option_data; try_again = opt->retry(); - PatternMatchData* pmd = opt->get_pattern(); + PatternMatchData* pmd = opt->get_pattern(0, RULE_WO_DIR); if ( pmd and pmd->last_check ) content_last = pmd->last_check + get_instance_id(); @@ -409,7 +410,7 @@ int detection_option_node_evaluate( switch ( node->option_type ) { case RULE_OPTION_TYPE_LEAF_NODE: - // Add the match for this otn to the queue. + // Add the match for this otn to the queue. { OptTreeNode* otn = (OptTreeNode*)node->option_data; int16_t app_proto = p->get_application_protocol(); @@ -486,8 +487,8 @@ int detection_option_node_evaluate( if ( content_last ) { if ( content_last->ts == p->pkth->ts && - content_last->packet_number == cur_eval_pkt_count && - content_last->rebuild_flag == (p->packet_flags & PKT_REBUILT_STREAM) ) + content_last->packet_number == cur_eval_pkt_count && + content_last->rebuild_flag == (p->packet_flags & PKT_REBUILT_STREAM) ) { rval = DETECTION_OPTION_NO_MATCH; break; @@ -520,7 +521,6 @@ int detection_option_node_evaluate( rval = node->evaluate(node->option_data, cursor, p); break; - } if ( rval == DETECTION_OPTION_NO_MATCH ) @@ -528,7 +528,6 @@ int detection_option_node_evaluate( state.last_check.result = result; return result; } - else if ( rval == DETECTION_OPTION_FAILED_BIT ) { eval_data->flowbit_failed = 1; @@ -537,7 +536,6 @@ int detection_option_node_evaluate( state.last_check.result = result; return 0; } - else if ( rval == DETECTION_OPTION_NO_ALERT ) { // Cache the current flowbit_noalert flag, and set it @@ -590,14 +588,13 @@ int detection_option_node_evaluate( continue; } } - else if ( child_node->option_type == RULE_OPTION_TYPE_CONTENT ) { // Check for an unbounded relative search. If this // failed before, it's going to fail again so don't // go down this path again IpsOption* opt = (IpsOption*)node->option_data; - PatternMatchData* pmd = opt->get_pattern(); + PatternMatchData* pmd = opt->get_pattern(0, RULE_WO_DIR); if ( pmd->unbounded() ) { @@ -610,7 +607,6 @@ int detection_option_node_evaluate( } } } - else if ( child_node->option_type == RULE_OPTION_TYPE_LEAF_NODE ) // Leaf node matched, don't eval again continue; @@ -665,12 +661,11 @@ int detection_option_node_evaluate( } if ( continue_loop && - rval == DETECTION_OPTION_MATCH && - node->relative_children ) + rval == DETECTION_OPTION_MATCH && + node->relative_children ) { continue_loop = try_again; } - else continue_loop = false; @@ -747,7 +742,6 @@ static void detection_option_node_update_otn_stats(detection_option_tree_node_t* local_stats.latency_timeouts = timeouts; local_stats.latency_suspends = suspends; } - else { local_stats.elapsed = node_stats.elapsed; @@ -812,7 +806,6 @@ void detection_option_tree_update_otn_stats(SFXHASH* doth) } } - detection_option_tree_root_t* new_root() { detection_option_tree_root_t* p = (detection_option_tree_root_t*) diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 04b2019f3..819da873b 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -471,7 +471,14 @@ bool set_fp_content(OptTreeNode* otn) fp_only = !ofl->ips_opt->fp_research(); } - PatternMatchData* tmp = get_pmd(ofl); + // Set rule direction + RuleDirection dir = RULE_WO_DIR; + if (OtnFlowFromServer(otn)) + dir = RULE_FROM_SERVER; + else if (OtnFlowFromClient(otn)) + dir = RULE_FROM_CLIENT; + + PatternMatchData* tmp = get_pmd(ofl, otn->proto, dir); if ( !tmp ) continue; @@ -537,7 +544,7 @@ static PatternMatchData* get_fp_content(OptTreeNode* otn, OptFpList*& next) if ( !ofl->ips_opt ) continue; - PatternMatchData* pmd = get_pmd(ofl); + PatternMatchData* pmd = get_pmd(ofl, 0, RULE_WO_DIR); if ( !pmd ) continue; @@ -585,8 +592,8 @@ static int fpFinishPortGroupRule( { static MpseAgent agent = { - pmx_create_tree, add_patrn_to_neg_list, - fpDeletePMX, free_detection_option_root, neg_list_free + pmx_create_tree, add_patrn_to_neg_list, + fpDeletePMX, free_detection_option_root, neg_list_free }; pg->mpse[pmd->pm_type] = MpseManager::get_search_engine( @@ -1924,7 +1931,6 @@ static void PrintFastPatternInfo(OptTreeNode* otn, PatternMatchData* pmd, snprintf(buf, sizeof(buf), "%2.02X ", (uint8_t)pattern[i]); hex += buf; txt += isprint(pattern[i]) ? pattern[i] : '.'; - } printf("fast pattern[%d] = x%s '%s'\n", pattern_length, hex.c_str(), txt.c_str()); #else diff --git a/src/framework/ips_option.h b/src/framework/ips_option.h index c6a6669c1..b31cea5a3 100644 --- a/src/framework/ips_option.h +++ b/src/framework/ips_option.h @@ -51,6 +51,13 @@ enum CursorActionType CAT_SET_KEY, }; +enum RuleDirection +{ + RULE_FROM_CLIENT, + RULE_FROM_SERVER, + RULE_WO_DIR +}; + class SO_PUBLIC IpsOption { public: @@ -78,7 +85,7 @@ public: { return CAT_NONE; } // for fast-pattern options like content - virtual struct PatternMatchData* get_pattern() + virtual struct PatternMatchData* get_pattern(int /*proto*/, RuleDirection) { return nullptr; } static int eval(void* v, Cursor& c, Packet* p) diff --git a/src/framework/range.cc b/src/framework/range.cc index 515b9bf04..d0ad2ea3f 100644 --- a/src/framework/range.cc +++ b/src/framework/range.cc @@ -177,7 +177,7 @@ void RangeCheck::init() min = max = 0; } -bool RangeCheck::is_set() +bool RangeCheck::is_set() const { return (op != MAX); } @@ -222,7 +222,7 @@ bool RangeCheck::parse(const char* s) return true; } -bool RangeCheck::eval(long c) +bool RangeCheck::eval(long c) const { switch ( op ) { diff --git a/src/framework/range.h b/src/framework/range.h index 9d81a9d1e..d7b330d13 100644 --- a/src/framework/range.h +++ b/src/framework/range.h @@ -48,10 +48,10 @@ public: bool operator==(const RangeCheck&) const; void init(); - bool is_set(); + bool is_set() const; // FIXIT-L add ttl style syntax bool parse(const char* s); - bool eval(long); + bool eval(long) const; }; #endif diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc index 55c31ddb2..f620c3064 100644 --- a/src/ips_options/ips_content.cc +++ b/src/ips_options/ips_content.cc @@ -143,7 +143,7 @@ public: int eval(Cursor& c, Packet*) override { return CheckANDPatternMatch(config, c); } - PatternMatchData* get_pattern() override + PatternMatchData* get_pattern(int, RuleDirection) override { return &config->pmd; } protected: @@ -243,10 +243,11 @@ static bool same_buffers( } return true; } + #endif // FIXIT-P fp, fp_only are set after hash table comparisons so this must -// return false to avoid unnecessary reevaluation and false positives. +// return this == &ips to avoid unnecessary reevaluation and false positives. // when this is fixed, add PatternMatchData::operator==(). bool ContentOption::operator==(const IpsOption& ips) const { @@ -286,10 +287,8 @@ bool ContentOption::operator==(const IpsOption& ips) const { return true; } -#else - UNUSED(ips); #endif - return false; + return this == &ips; } //------------------------------------------------------------------------- @@ -410,7 +409,6 @@ static int CheckANDPatternMatch(ContentData* idx, Cursor& c) DebugMessage(DEBUG_PATTERN_MATCH, "Pattern match found\n"); return DETECTION_OPTION_MATCH; } - else { DebugMessage(DEBUG_PATTERN_MATCH, "Pattern match failed\n"); diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index 87f6f63b7..73e0e82c4 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -96,7 +96,7 @@ public: bool retry() override; - PatternMatchData* get_pattern() override + PatternMatchData* get_pattern(int, RuleDirection) override { return &config.pmd; } int eval(Cursor&, Packet*) override; @@ -137,7 +137,7 @@ uint32_t RegexOption::hash() const return c; } -// see ContentOption::operator==() for why this is always false +// see ContentOption::operator==() bool RegexOption::operator==(const IpsOption& ips) const { #if 0 @@ -146,15 +146,12 @@ bool RegexOption::operator==(const IpsOption& ips) const RegexOption& rhs = (RegexOption&)ips; - if ( config.re == rhs.config.re and - config.pmd.flags == rhs.config.pmd.flags and - config.pmd.relative == rhs.config.pmd.relative ) + if ( config.re == rhs.config.re and + config.pmd.flags == rhs.config.pmd.flags and + config.pmd.relative == rhs.config.pmd.relative ) return true; - -#else - UNUSED(ips); #endif - return false; + return this == &ips; } static int hs_match( @@ -268,7 +265,6 @@ bool RegexModule::set(const char*, Value& v, SnortConfig*) config.re.erase(0, 1); config.re.erase(config.re.length()-1, 1); } - else if ( v.is("nocase") ) { config.pmd.flags |= HS_FLAG_CASELESS; diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index c595faa8b..62be4b6e7 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -81,7 +81,7 @@ struct SdPatternConfig std::string pii; unsigned threshold = 1; bool obfuscate_pii = false; - int (*validate)(const uint8_t* buf, unsigned long long buflen) = nullptr; + int (* validate)(const uint8_t* buf, unsigned long long buflen) = nullptr; inline bool operator==(const SdPatternConfig& rhs) const { @@ -119,7 +119,7 @@ public: uint32_t hash() const override; bool operator==(const IpsOption&) const override; - PatternMatchData* get_pattern() override + PatternMatchData* get_pattern(int, RuleDirection) override { return &config.pmd; } int eval(Cursor&, Packet* p) override; @@ -136,7 +136,7 @@ SdPatternOption::SdPatternOption(const SdPatternConfig& c) : { // FIXIT-L why is this failing but everything is working? ParseError("can't initialize sd_pattern for %s (%d) %p", - config.pii.c_str(), err, (void*)s_scratch); + config.pii.c_str(), err, (void*)s_scratch); } config.pmd.pattern_buf = config.pii.c_str(); @@ -146,7 +146,7 @@ SdPatternOption::SdPatternOption(const SdPatternConfig& c) : } SdPatternOption::~SdPatternOption() -{ +{ if ( config.db ) hs_free_database(config.db); } @@ -173,10 +173,10 @@ bool SdPatternOption::operator==(const IpsOption& ips) const return false; } -struct hsContext +struct hsContext { - hsContext(const SdPatternConfig &c_, Packet* p_, const uint8_t* const start_) - : config(c_), packet(p_), start(start_) {} + hsContext(const SdPatternConfig& c_, Packet* p_, const uint8_t* const start_) + : config(c_), packet(p_), start(start_) { } unsigned int count = 0; @@ -209,7 +209,7 @@ static int hs_match(unsigned int /*id*/, unsigned long long from, ctx->packet->obfuscator = new Obfuscator(); uint32_t off = ctx->buf - ctx->start; - // FIXIT-L Make configurable or don't show any PII partials (0 for user defined??) + // FIXIT-L Make configurable or don't show any PII partials (0 for user defined??) len = len > 4 ? len - 4 : len; ctx->packet->obfuscator->push(off, len); } @@ -323,13 +323,11 @@ bool SdPatternModule::set(const char*, Value& v, SnortConfig* sc) config.validate = SdLuhnAlgorithm; config.obfuscate_pii = sc->obfuscate_pii; } - else if (config.pii == "us_social") { config.pii = SD_SOCIAL_PATTERN; config.obfuscate_pii = sc->obfuscate_pii; } - else if (config.pii == "us_social_nodashes") { config.pii = SD_SOCIAL_NODASHES_PATTERN; @@ -343,7 +341,8 @@ bool SdPatternModule::end(const char*, int, SnortConfig*) { hs_compile_error_t* err = nullptr; - if ( hs_compile(config.pii.c_str(), HS_FLAG_DOTALL|HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, nullptr, &config.db, &err) + if ( hs_compile(config.pii.c_str(), HS_FLAG_DOTALL|HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, + nullptr, &config.db, &err) or !config.db ) { ParseError("can't compile regex '%s'", config.pii.c_str()); @@ -407,7 +406,7 @@ static IpsOption* sd_pattern_ctor(Module* m, OptTreeNode*) } static void sd_pattern_dtor(IpsOption* p) -{ +{ delete p; } diff --git a/src/parser/parse_rule.cc b/src/parser/parse_rule.cc index d0c45ea23..c93c5e00e 100644 --- a/src/parser/parse_rule.cc +++ b/src/parser/parse_rule.cc @@ -816,7 +816,7 @@ static void XferHeader(RuleTreeNode* test_node, RuleTreeNode* rtn) * ***************************************************************************/ static void AddRuleFuncToList( - int (* rfunc) (Packet*, RuleTreeNode*, struct RuleFpList*, int), + int (* rfunc)(Packet*, RuleTreeNode*, struct RuleFpList*, int), RuleTreeNode* rtn) { RuleFpList* idx; @@ -1143,17 +1143,17 @@ static int mergeDuplicateOtn( return 1; } -PatternMatchData* get_pmd(OptFpList* ofl) +PatternMatchData* get_pmd(OptFpList* ofl, int proto, RuleDirection direction) { if ( !ofl->ips_opt ) return nullptr; - return ofl->ips_opt->get_pattern(); + return ofl->ips_opt->get_pattern(proto, direction); } static void finalize_content(OptFpList* ofl) { - PatternMatchData* pmd = get_pmd(ofl); + PatternMatchData* pmd = get_pmd(ofl, 0, RULE_WO_DIR); if ( !pmd ) return; @@ -1165,7 +1165,7 @@ static void finalize_content(OptFpList* ofl) bool is_fast_pattern_only(OptFpList* ofl) { - PatternMatchData* pmd = get_pmd(ofl); + PatternMatchData* pmd = get_pmd(ofl, 0, RULE_WO_DIR); if ( !pmd ) return false; @@ -1175,7 +1175,7 @@ bool is_fast_pattern_only(OptFpList* ofl) static void clear_fast_pattern_only(OptFpList* ofl) { - PatternMatchData* pmd = get_pmd(ofl); + PatternMatchData* pmd = get_pmd(ofl, 0, RULE_WO_DIR); if ( pmd && pmd->fp_only > 0 ) pmd->fp_only = 0; @@ -1199,7 +1199,7 @@ static void ValidateFastPattern(OptTreeNode* otn) } // reset the check if one of these are present. - if ( fpl->ips_opt and !fpl->ips_opt->get_pattern() ) + if ( fpl->ips_opt and !fpl->ips_opt->get_pattern(0, RULE_WO_DIR)) { if ( fpl->ips_opt->get_cursor_type() > CAT_NONE ) relative_is_bad_mkay = false; @@ -1536,7 +1536,7 @@ const char* parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* ot * After otn processing we can finalize port object processing for this rule */ if ( FinishPortListRule( - sc->port_tables, new_rtn, otn, rtn.proto, has_fp, sc->fast_pattern_config) ) + sc->port_tables, new_rtn, otn, rtn.proto, has_fp, sc->fast_pattern_config) ) ParseError("Failed to finish a port list rule."); return nullptr; diff --git a/src/parser/parse_rule.h b/src/parser/parse_rule.h index dcd535fda..20b23a05e 100644 --- a/src/parser/parse_rule.h +++ b/src/parser/parse_rule.h @@ -21,6 +21,7 @@ #define PARSE_RULE_H #include "detection/rules.h" +#include "framework/ips_option.h" struct OptFpList; struct OptTreeNode; @@ -44,7 +45,7 @@ OptTreeNode* parse_rule_open(SnortConfig*, RuleTreeNode&, bool stub = false); const char* parse_rule_close(SnortConfig*, RuleTreeNode&, OptTreeNode*); bool is_fast_pattern_only(OptFpList*); -struct PatternMatchData* get_pmd(OptFpList*); +struct PatternMatchData* get_pmd(OptFpList*, int proto, RuleDirection); int get_rule_count(); diff --git a/src/service_inspectors/dce_rpc/ips_dce_iface.cc b/src/service_inspectors/dce_rpc/ips_dce_iface.cc index 765e661e4..72aca4631 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_iface.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_iface.cc @@ -29,8 +29,11 @@ #include "framework/range.h" #include "detection/detect.h" #include "detection/detection_defines.h" +#include "detection/pattern_match_data.h" #include "hash/sfhashfcn.h" #include "profiler/profiler.h" +#include "target_based/snort_protocols.h" +#include "main/snort_debug.h" //------------------------------------------------------------------------- // dcerpc2 interface rule options @@ -199,19 +202,72 @@ class Dce2IfaceOption : public IpsOption { public: Dce2IfaceOption(RangeCheck iface_version, bool iface_any_frag, Uuid iface_uuid) : - IpsOption(s_name) - { version = iface_version; any_frag = iface_any_frag; uuid = iface_uuid; } + IpsOption(s_name), version(iface_version), any_frag(iface_any_frag), uuid(iface_uuid) + { + memset(&pmd, 0, sizeof(pmd)); + } uint32_t hash() const override; bool operator==(const IpsOption&) const override; int eval(Cursor&, Packet*) override; + PatternMatchData* get_pattern(int proto, RuleDirection direction) override; + ~Dce2IfaceOption(); private: - RangeCheck version; - bool any_frag; - Uuid uuid; + const RangeCheck version; + const bool any_frag; + const Uuid uuid; + PatternMatchData pmd; }; +Dce2IfaceOption::~Dce2IfaceOption() +{ + if ( pmd.pattern_buf) + { + snort_free((char*)pmd.pattern_buf); + } +} + +PatternMatchData* Dce2IfaceOption::get_pattern(int proto, RuleDirection direction) +{ + if (pmd.pattern_buf) + { + return &pmd; + } + + if (proto == SNORT_PROTO_TCP) + { + const char client_fp[] = "\x05\x00\x00"; + const char server_fp[] = "\x05\x00\x02"; + const char no_dir_fp[] = "\x05\x00"; + + switch (direction) + { + case RULE_FROM_CLIENT: + pmd.pattern_size = 3; + pmd.pattern_buf = (char*)snort_alloc(pmd.pattern_size); + memcpy((void*)pmd.pattern_buf, client_fp, pmd.pattern_size); + break; + + case RULE_FROM_SERVER: + pmd.pattern_size = 3; + pmd.pattern_buf = (char*)snort_alloc(pmd.pattern_size); + memcpy((void*)pmd.pattern_buf, server_fp, pmd.pattern_size); + break; + + default: + pmd.pattern_size = 2; + pmd.pattern_buf = (char*)snort_alloc(pmd.pattern_size); + memcpy((void*)pmd.pattern_buf, no_dir_fp, pmd.pattern_size); + break; + } + return &pmd; + } + // FIXIT-L add udp fast pattern + + return nullptr; +} + uint32_t Dce2IfaceOption::hash() const { uint32_t a, b, c; @@ -244,19 +300,11 @@ uint32_t Dce2IfaceOption::hash() const bool Dce2IfaceOption::operator==(const IpsOption& ips) const { - if ( strcmp(get_name(), ips.get_name()) ) - return false; - - const Dce2IfaceOption& rhs = (Dce2IfaceOption&)ips; - - if ((DCE2_UuidCompare(&uuid, &rhs.uuid) == 0) && - (version == rhs.version) && - (any_frag == rhs.any_frag)) - { - return true; - } - - return false; + // FIXIT-L + // Fast pattern is calculated only after the entire rule is parsed. + // The rule option can be mistaken as a duplicate because we don't take the fast pattern into + // account. Instead of comparing values, make sure it is the same object. + return this == &ips; } int Dce2IfaceOption::eval(Cursor&, Packet* p)