Merge pull request #1439 in SNORT/snort3 from ~RUCOMBS/snort3:rule_stubs to master

Squashed commit of the following:

commit bc201990e97b748a9a023687640150b0c1d7274d
Author: russ <rucombs@cisco.com>
Date:   Sat Nov 17 09:32:47 2018 -0500

    so rules: add robust stub parsing

diff --git a/src/managers/so_manager.cc b/src/managers/so_manager.cc
index b4e96512d90a2a4fbccbe38f1d45ed1faad0838d..3f7f969bf454806b5284e442713c7233e549010c 100644 (file)
--- a/src/managers/so_manager.cc
+++ b/src/managers/so_manager.cc
@@ -34,6 +34,7 @@
 #include <sstream>
 
 #include "log/messages.h"
+#include "parser/parse_so_rule.h"
 
 using namespace std;
 
@@ -200,21 +201,19 @@ const char* SoManager::get_so_options(const char* soid)
     if ( !api )
         return nullptr;
 
-    if ( !api->length )
-        return ")";  // plain stub is full rule
-
     const char* rule = revert(api->rule, api->length);
 
     if ( !rule )
         return nullptr;
 
-    // FIXIT-L this approach won't tolerate spaces and might get
-    // fooled by matching content (should it precede this)
-    char opt[32];
-    snprintf(opt, sizeof(opt), "soid:%s;", soid);
-    const char* s = strstr(rule, opt);
+    static std::string opts;
+    opts.clear();
+
+    if ( !::get_so_options(rule, !api->length, opts) )
+        return nullptr;
 
-    return s ? s + strlen(opt) : nullptr;
+    opts += " )";
+    return opts.c_str();
 }
 
 SoEvalFunc SoManager::get_so_eval(const char* soid, const char* so, void** data)
@@ -243,34 +242,17 @@ void SoManager::dump_rule_stubs(const char*)
 
     for ( auto* p : s_rules )
     {
-        const char* s;
         const char* rule = revert(p->rule, p->length);
 
         if ( !rule )
             continue;
 
-        // FIXIT-L need to properly parse rule to avoid
-        // confusing other text for soid option
-        if ( !(s = strstr(rule, "soid:")) )
-            continue;
+        std::string stub;
 
-        if ( !(s = strchr(s, ';')) )
+        if ( !get_so_stub(rule, !p->length, stub) )
             continue;
 
-        if ( *rule == '\n' )
-            ++rule;
-
-        unsigned n = p->length ? s-rule+1 : strlen(rule);
-
-        if ( n and rule[n-1] == '\n' )
-            --n;
-
-        cout.write(rule, n);
-
-        if ( p->length )
-            cout << " )";
-
-        cout << endl;
+        cout << stub << endl;
 
         ++c;
     }

diff --git a/src/parser/CMakeLists.txt b/src/parser/CMakeLists.txt
index ff81659a0b0612e992b24cb8cabe942b4efcf48d..479045e91a5ee8faffa10b4ee5308622b1373119 100644 (file)
--- a/src/parser/CMakeLists.txt
+++ b/src/parser/CMakeLists.txt
@@ -12,6 +12,8 @@ add_library (parser OBJECT
     parse_ports.h
     parse_rule.cc
     parse_rule.h
+    parse_so_rule.cc
+    parse_so_rule.h
     parse_stream.cc
     parse_stream.h
     parse_utils.cc

diff --git a/src/parser/parse_so_rule.cc b/src/parser/parse_so_rule.cc
new file mode 100644 (file)
index 0000000..98e1776
--- /dev/null
+++ b/src/parser/parse_so_rule.cc
@@ -0,0 +1,456 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2018 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// parse_so_rule.cc author Russ Combs <rucombs@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "parse_so_rule.h"
+
+#include <cctype>
+#include <set>
+#include <string>
+
+#ifdef UNIT_TEST
+#include "catch/snort_catch.h"
+#endif
+
+// must parse out stub options for --dump-dynamic-rules
+// must parse out detection options (ie everything else) after loading stub
+//
+// for plain so rules, all options are in stub
+// for protected rules, stub options depend on the use of UNORDERED_OPTS
+//
+// assume valid rule syntax
+// handles # and /* */ comments
+// return true if parsed rule body close
+// no requirement to beautify ugly rules
+
+class SoRuleParser
+{
+public:
+    SoRuleParser(bool p)
+    { is_plain = p; }
+
+    bool parse_so_rule(const char* in, std::string& stub, std::string& opts);
+
+private:
+    bool is_stub_option(std::string& opt);
+
+private:
+    bool in_stub;
+    bool is_plain;
+};
+
+//#define UNORDERED_OPTS
+#ifdef UNORDERED_OPTS
+// these options are shown in so rule stubs
+// any other option is considered a detection option
+static std::set<std::string> stub_opts =
+{
+    "classtype", "flowbits", "gid", "metadata", "msg", "priority",
+    "reference", "rem", "rev", "service", "sid", "soid"
+};
+
+bool SoRuleParser::is_stub_option(std::string& opt)
+{
+    if ( is_plain )
+        return true;
+
+    size_t n = opt.find_first_of(" :;");
+    std::string name = opt.substr(0, n);
+    return stub_opts.find(name) != stub_opts.end();
+}
+#else
+// all options up to and including soid are shown in so rule stubs
+// any options following soid are considered detection options
+// this approach requires Talos to reorder rule options so is not
+// viable long-term.
+bool SoRuleParser::is_stub_option(std::string& opt)
+{
+    if ( is_plain )
+        return true;
+
+    if ( !in_stub )
+        return false;
+
+    size_t n = opt.find_first_of(" :;");
+    std::string name = opt.substr(0, n);
+
+    if  ( name == "soid" )
+    {
+        in_stub = false;
+        return true;
+    }
+    return true;
+}
+#endif
+
+// split rule into stub and detection options
+bool SoRuleParser::parse_so_rule(const char* in, std::string& stub, std::string& opts)
+{
+    in_stub = true;
+
+    int state = 0;
+    int next = 0;
+
+    bool del_sp = false;
+
+    std::string opt;
+    std::string* accum = &stub;
+
+    while ( *in )
+    {
+        switch ( state )
+        {
+        case 0:
+            if ( *in == '#' )
+            {
+                state = 1;
+                next = 0;
+                break;
+            }
+            else if ( *in == '(' )
+                state = 5;
+            else if ( *in == '/' )
+            {
+                state = 2;
+                next = 0;
+            }
+            break;
+        case 1:
+            if ( *in == '\n' )
+                state = next;
+            break;
+        case 2:
+            if ( *in == '*' )
+                state = 3;
+            else
+            {
+                state = next;
+                continue; // repeat
+            }
+            break;
+        case 3:
+            if ( *in == '*' )
+                state = 4;
+            break;
+        case 4:
+            if ( *in == '/' )
+                state = next;
+            else
+                state = 3;
+            break;
+        case 5:
+            if ( del_sp )
+            {
+                if ( std::isspace(*in) )
+                {
+                    opts += *in++;
+                    continue;
+                }
+                else
+                    del_sp = false;
+            }
+            if ( *in == '#' )
+            {
+                state = 1;
+                next = 5;
+                break;
+            }
+            else if ( *in == '/' )
+            {
+                state = 2;
+                next = 5;
+            }
+            else if ( *in == ')' )
+            {
+                *accum += *in;
+                return true;
+            }
+            else if ( !std::isspace(*in) )
+            {
+                opt.clear();
+                accum = &opt;
+                state = 6;
+            }
+            break;
+        case 6:
+            if ( *in == '#' )
+            {
+                state = 1;
+                next = 6;
+                break;
+            }
+            else if ( *in == '/' )
+            {
+                state = 2;
+                next = 6;
+            }
+            else if ( *in == '"' )
+            {
+                state = 7;
+            }
+            else if ( *in == ';' )
+            {
+                accum = &stub;
+                state = 5;
+                
+                if ( is_stub_option(opt) )
+                    stub += opt;
+                else
+                {
+                    opts += opt;
+                    opts += *in++;
+                    del_sp = true;
+                    continue;
+                }
+            }
+            break;
+        case 7:
+            if ( *in == '\\' )
+                state = 8;
+            else if ( *in == '"' )
+                state = 6;
+            break;
+        case 8:
+            state = 7;
+            break;
+        }
+        *accum += *in++;
+    }
+
+    return false;
+}
+
+//--------------------------------------------------------------------------
+// public methods
+//--------------------------------------------------------------------------
+
+bool get_so_stub(const char* in, bool plain, std::string& stub)
+{
+    SoRuleParser sop(plain);
+    std::string opts;
+    return sop.parse_so_rule(in, stub, opts);
+}
+
+bool get_so_options(const char* in, bool plain, std::string& opts)
+{
+    SoRuleParser sop(plain);
+    std::string stub;
+    return sop.parse_so_rule(in, stub, opts);
+}
+
+//--------------------------------------------------------------------------
+// test data
+//--------------------------------------------------------------------------
+
+#ifdef UNIT_TEST
+struct TestCase
+{
+    const char* rule;
+    const char* expect;
+    bool result;
+};
+
+static const TestCase syntax_tests[] =
+{
+    { "alert() ", "alert()", true },
+
+    { "alert tcp any any -> any any ( )", 
+      "alert tcp any any -> any any ( )", true },
+
+    { "alert tcp $EXTERNAL_NET any -> $HTTP_SERVERS $HTTP_PORTS ( )", 
+      "alert tcp $EXTERNAL_NET any -> $HTTP_SERVERS $HTTP_PORTS ( )", true },
+
+    { "#alert()", "#alert()", false },
+    { "# \nalert()", "# \nalert()", true },
+    { "alert#\n()", "alert#\n()", true },
+    { "alert() # comment", "alert()", true },
+    { "alert(#\n)", "alert(#\n)", true },
+    { "alert(#)", "alert(#)", false },
+
+    { "/*alert()*/", "/*alert()*/", false },
+    { "/ *alert()*/", "/ *alert()", true },
+    { "/* /alert()", "/* /alert()", false },
+    { "/* *alert()", "/* *alert()", false },
+    { "/*alert(*/)", "/*alert(*/)", false },
+    { "alert(/*)", "alert(/*)", false },
+    { "alert(/*)*/", "alert(/*)*/", false },
+    { "alert(/**)/", "alert(/**)/", false },
+    { "alert/*()*/", "alert/*()*/", false },
+    { "alert/*(*/)", "alert/*(*/)", false },
+
+    { "alert(/**/) ", "alert(/**/)", true },
+    { "alert(/* sid:1; */)", "alert(/* sid:1; */)", true },
+
+    { "alert( sid:1; )", "alert( sid:1; )", true },
+
+    { "alert( sid:1 /*comment*/; )", "alert( sid:1 /*comment*/; )", true },
+    { "alert( sid:1 # comment\n; )", "alert( sid:1 # comment\n; )", true },
+    { "alert( sid:1; /*id:0;*/ )", "alert( sid:1; /*id:0;*/ )", true },
+
+    { nullptr, nullptr, false }
+};
+
+// __STRDUMP_DISABLE__
+static const TestCase stub_tests[] =
+{
+#ifdef UNORDERED_OPTS
+    { "alert( id:0; )", "alert( )", true },
+    { "alert( sid:1; id:0; )", "alert( sid:1; )", true },
+    { "alert( sid:1;id:0; )", "alert( sid:1;)", true },
+    { "alert( id:0;sid:1; )", "alert( sid:1; )", true },
+
+    { "alert( id:/*comment*/0; )", "alert( )", true },
+    { "alert( id: #comment\n0; )", "alert( )", true },
+
+    { R"_(alert( content:"foo"; ))_", "alert( )", true },
+    { R"_(alert( content:"f;o"; ))_", "alert( )", true },
+    { R"_(alert( content:"f\"o"; ))_", "alert( )", true },
+
+    { R"_(alert( soid; rem:"soid"; /*soid*/ ))_",
+      R"_(alert( soid; rem:"soid"; /*soid*/ ))_", true },
+
+    { R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; metadata:impact_flag red,policy balanced-ips drop,policy security-ips drop; service:http; reference:url,www.virustotal.com/file/bff436d8a2ccf1cdce56faabf341e97f59285435b5e73f952187bbfaf4df3396/analysis/; classtype:trojan-activity; sid:24062; rev:7; ))_", 
+      R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; metadata:impact_flag red,policy balanced-ips drop,policy security-ips drop; service:http; reference:url,www.virustotal.com/file/bff436d8a2ccf1cdce56faabf341e97f59285435b5e73f952187bbfaf4df3396/analysis/; classtype:trojan-activity; sid:24062; rev:7; ))_", 
+      true },
+
+#else
+    { "alert( soid; id:0; )", "alert( soid; )", true },
+    { "alert( sid:1; soid; id:0; )", "alert( sid:1; soid; )", true },
+    { "alert( sid:1;soid;id:0; )", "alert( sid:1;soid;)", true },
+    { "alert( soid;id:0;sid:1; )", "alert( soid;)", true },
+
+    { "alert( soid; id:/*comment*/0; )", "alert( soid; )", true },
+    { "alert( soid; id: #comment\n0; )", "alert( soid; )", true },
+
+    { R"_(alert( soid; content:"foo"; ))_", "alert( soid; )", true },
+    { R"_(alert( soid; content:"f;o"; ))_", "alert( soid; )", true },
+    { R"_(alert( soid; content:"f\"o"; ))_", "alert( soid; )", true },
+
+    { R"_(alert( rem:"soid"; /*soid*/ soid; ))_",
+      R"_(alert( rem:"soid"; /*soid*/ soid; ))_", true },
+
+    { R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; soid:a; flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; metadata:impact_flag red,policy balanced-ips drop,policy security-ips drop; service:http; reference:url,www.virustotal.com/file/bff436d8a2ccf1cdce56faabf341e97f59285435b5e73f952187bbfaf4df3396/analysis/; classtype:trojan-activity; sid:24062; rev:7; ))_", 
+      R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; soid:a; ))_", 
+      true },
+
+#endif
+    { nullptr, nullptr, false }
+};
+
+static const TestCase opts_tests[] =
+{
+#ifdef UNORDERED_OPTS
+    { R"_(alert( soid; rem:"soid"; /*soid*/ ))_", R"_(alert( /*soid*/ ))_", true },
+
+    { R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; metadata:impact_flag red,policy balanced-ips drop,policy security-ips drop; service:http; reference:url,www.virustotal.com/file/bff436d8a2ccf1cdce56faabf341e97f59285435b5e73f952187bbfaf4df3396/analysis/; classtype:trojan-activity; sid:24062; rev:7; ))_", 
+      R"_(flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; )_", 
+      true },
+
+#else
+    { R"_(alert( rem:"soid"; /*soid*/ soid; ))_", "", true },
+
+    { R"_(alert tcp $HOME_NET any -> $EXTERNAL_NET $HTTP_PORTS ( msg:"MALWARE-CNC Win.Trojan.Hufysk variant outbound connection"; metadata:impact_flag red,policy balanced-ips drop,policy security-ips drop; service:http; reference:url,www.virustotal.com/file/bff436d8a2ccf1cdce56faabf341e97f59285435b5e73f952187bbfaf4df3396/analysis/; classtype:trojan-activity; sid:24062; rev:7; soid:3_24062_7; flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; ))_", 
+      R"_(flow:to_server,established; http_uri; content:"/j.php|3F|u|3D|", fast_pattern,nocase; content:"&v=f2&r=",depth 8,offset 41,nocase; )_", 
+      true },
+#endif
+    { nullptr, nullptr, false }
+};
+// __STRDUMP_ENABLE__
+
+//--------------------------------------------------------------------------
+// unit tests
+//--------------------------------------------------------------------------
+
+TEST_CASE("parse_so_rule", "[parser]")
+{
+    const TestCase* tc = syntax_tests;
+
+    while ( tc->rule )
+    {
+        SoRuleParser sop(false);
+        std::string stub;
+        std::string opts;
+        bool parse = sop.parse_so_rule(tc->rule, stub, opts);
+        CHECK(parse == tc->result);
+        CHECK(stub == tc->expect);
+        ++tc;
+    }
+}
+
+TEST_CASE("get_so_stub protected", "[parser]")
+{
+    const TestCase* tc = stub_tests;
+
+    while ( tc->rule )
+    {
+        std::string stub;
+        bool get = get_so_stub(tc->rule, false, stub);
+        CHECK(get == tc->result);
+        CHECK(stub == tc->expect);
+        ++tc;
+    }
+}
+
+TEST_CASE("get_so_options protected", "[parser]")
+{
+    const TestCase* tc = opts_tests;
+
+    while ( tc->rule )
+    {
+        std::string opts;
+        bool get = get_so_options(tc->rule, false, opts);
+        CHECK(get == tc->result);
+        CHECK(opts == tc->expect);
+        ++tc;
+    }
+}
+
+TEST_CASE("get_so_stub plain", "[parser]")
+{
+    const TestCase* tc = stub_tests;
+
+    while ( tc->rule )
+    {
+        std::string stub;
+        bool get = get_so_stub(tc->rule, true, stub);
+        CHECK(get == tc->result);
+        CHECK(stub == tc->rule);
+        ++tc;
+    }
+}
+
+TEST_CASE("get_so_options plain", "[parser]")
+{
+    const TestCase* tc = opts_tests;
+
+    while ( tc->rule )
+    {
+        std::string opts;
+        bool get = get_so_options(tc->rule, true, opts);
+        CHECK(get == tc->result);
+        CHECK(opts == "");
+        ++tc;
+    }
+}
+#endif
+

diff --git a/src/parser/parse_so_rule.h b/src/parser/parse_so_rule.h
new file mode 100644 (file)
index 0000000..3e815ca
--- /dev/null
+++ b/src/parser/parse_so_rule.h
@@ -0,0 +1,28 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2018 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+#ifndef PARSE_SO_RULE_H
+#define PARSE_SO_RULE_H
+
+#include <string>
+
+bool get_so_stub(const char* in, bool plain, std::string& opts);
+bool get_so_options(const char* in, bool plain, std::string& opts);
+
+#endif
+