From: Russ Combs (rucombs) Date: Tue, 28 Apr 2020 14:52:46 +0000 (+0000) Subject: Merge pull request #2181 in SNORT/snort3 from ~RUCOMBS/snort3:more_meta to master X-Git-Tag: 3.0.1-3~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=49af2b2c52ef9a69bf798ef95839c0aeb767e102;p=thirdparty%2Fsnort3.git Merge pull request #2181 in SNORT/snort3 from ~RUCOMBS/snort3:more_meta to master Squashed commit of the following: commit aac87fdd266361917e23a8f4490eaadbdd4a72b7 Author: russ Date: Sat Apr 25 12:20:02 2020 -0400 so rules: allow #fragments in references in so rule stubs Disallow # comments within so rule stub options since #frags in references were interpreted as comments. Need to refactor the main parser to support this case. commit 41e61ea2f0639ff68fd85e4989d4e5b83b40dc60 Author: russ Date: Fri Apr 24 17:28:52 2020 -0400 parameter: reject reals assigned to ints commit f7b6c8b83ec5609f92d4b270a3d4c53db064cd6b Author: russ Date: Wed Apr 22 16:46:38 2020 -0400 snort: convert --dump-rule-{meta,state,deps} to json format commit 113228ee427c78785959445c2e56eb376c0e5478 Author: russ Date: Thu Apr 23 09:46:12 2020 -0400 json: add stream formatter helper commit 5a47d3ea423fe3dccdd7045b603fbfae01a09250 Author: russ Date: Wed Apr 22 13:06:31 2020 -0400 snort: add classtype, priority, and references to --dump-rule-meta output --- diff --git a/src/detection/signature.cc b/src/detection/signature.cc index 62ffe74f0..5f8457785 100644 --- a/src/detection/signature.cc +++ b/src/detection/signature.cc @@ -31,6 +31,7 @@ #include "framework/decode_data.h" #include "hash/hash_defs.h" #include "hash/ghash.h" +#include "helpers/json_stream.h" #include "ips_options/ips_flowbits.h" #include "log/messages.h" #include "main/snort_config.h" @@ -229,6 +230,10 @@ void OtnLookupFree(GHash* otn_map) delete otn_map; } +//-------------------------------------------------------------------------- +// dump msg map +//-------------------------------------------------------------------------- + void dump_msg_map(const SnortConfig* sc) { GHashNode* ghn = sc->otn_map->find_first(); @@ -251,6 +256,10 @@ void dump_msg_map(const SnortConfig* sc) } } +//-------------------------------------------------------------------------- +// dump rule meta +//-------------------------------------------------------------------------- + static void get_flow_bits( const OptTreeNode* otn, std::vector& setters, std::vector& checkers) { @@ -276,89 +285,133 @@ static void get_flow_bits( } } -static void dump_field(const char* key, long val, bool sep = true) -{ if ( sep ) std::cout << ", "; std::cout << key << ": " << val; } +static void dump_sid(JsonStream& j, const SigInfo& si) +{ + j.put("gid", si.gid); + j.put("sid", si.sid); + j.put("rev", si.rev); +} -static void dump_field(const char* key, const std::string& val, bool sep = true) -{ if ( sep ) std::cout << ", "; std::cout << key << ": " << val; } +static void dump_header(JsonStream& j, const RuleHeader* h) +{ + assert(h); + j.put("action", h->action); + j.put("src_nets", h->src_nets); + j.put("src_ports", h->src_ports); + j.put("direction", h->dir); + j.put("dst_nets", h->dst_nets); + j.put("dst_ports", h->dst_ports); +} -static void dump_opt(const char* key, const std::string& val, bool sep = true) +static void dump_info(JsonStream& j, const SigInfo& si) { - if ( val.empty() ) + if ( si.class_type ) + j.put("classtype", si.class_type->name); + + j.put("priority", si.priority); + + size_t n = si.message.length(); + assert(n > 2 and si.message[0] == '"' and si.message[n-1] == '"'); + std::string msg = si.message.substr(1, n-2); + j.put("msg", msg); +} + +static void dump_services(JsonStream& json, const SigInfo& si) +{ + if ( si.services.empty() ) return; - if ( sep ) - std::cout << ", "; + json.open_array("services"); + + for ( const auto& svc : si.services ) + json.put(nullptr, svc.service); - std::cout << key << ": " << val; + json.close_array(); } -static void dump_info(const SigInfo& si) +static void dump_bits(JsonStream& json, const char* key, std::vector& bits) { - dump_field("gid", si.gid, false); - dump_field("sid", si.sid); - dump_field("rev", si.rev); + if ( bits.empty() ) + return; + + json.open_array(key); + + for ( const auto& s : bits ) + json.put(nullptr, s); + + json.close_array(); } -static void dump_header(const RuleHeader* h) +static void dump_refs(JsonStream& json, const SigInfo& si) { - assert(h); - dump_opt("action", h->action); - dump_opt("src_nets", h->src_nets); - dump_opt("src_ports", h->src_ports); - dump_opt("direction", h->dir); - dump_opt("dst_nets", h->dst_nets); - dump_opt("dst_ports", h->dst_ports); + if ( si.refs.empty() ) + return; + + json.open_array("references"); + + for ( const auto& rn : si.refs ) + { + json.open(); + json.put("system", rn->system->name); + json.put("id", rn->id); + json.close(); + } + + json.close_array(); } void dump_rule_meta(const SnortConfig* sc) { GHashNode* ghn = sc->otn_map->find_first(); + JsonStream json(std::cout); while ( ghn ) { const OptTreeNode* otn = (OptTreeNode*)ghn->data; - const SigInfo& si = otn->sigInfo; - - dump_info(si); - const RuleTreeNode* rtn = otn->proto_nodes[0]; - dump_header(rtn->header); + const SigInfo& si = otn->sigInfo; - dump_field("msg", si.message); + json.open(); - for ( const auto& svc : si.services ) - dump_field("service", svc.service); + dump_sid(json, si); + dump_header(json, rtn->header); + dump_info(json, si); + dump_services(json, si); std::vector setters; std::vector checkers; get_flow_bits(otn, setters, checkers); - for ( const auto& s : setters ) - dump_field("sets", s); + dump_bits(json, "sets", setters); + dump_bits(json, "checks", checkers); + dump_refs(json, si); - for ( const auto& s : checkers ) - dump_field("checks", s); + json.put("body", *si.body); + json.close(); - dump_field("body", *si.body); - - std::cout << std::endl; ghn = sc->otn_map->find_next(); } } +//-------------------------------------------------------------------------- +// dump rule states +//-------------------------------------------------------------------------- + void dump_rule_state(const SnortConfig* sc) { GHashNode* ghn = sc->otn_map->find_first(); + JsonStream json(std::cout); while ( ghn ) { const OptTreeNode* otn = (OptTreeNode*)ghn->data; const SigInfo& si = otn->sigInfo; - dump_field("gid", si.gid, false); - dump_field("sid", si.sid); - dump_field("rev", si.rev); + json.open(); + json.put("gid", si.gid); + json.put("sid", si.sid); + json.put("rev", si.rev); + json.open_array("states"); for ( unsigned i = 0; i < otn->proto_node_num; ++i ) { @@ -367,20 +420,30 @@ void dump_rule_state(const SnortConfig* sc) if ( !rtn ) continue; + json.open(); + auto pid = snort::get_ips_policy(sc, i)->user_policy_id; - dump_field("policy", pid); + json.put("policy", pid); const char* s = Actions::get_string(rtn->action); - dump_field("action", s); + json.put("action", s); s = rtn->enabled() ? "enabled" : "disabled"; - dump_field("state", s); + json.put("state", s); + + json.close(); } - std::cout << std::endl; + json.close_array(); + json.close(); + ghn = sc->otn_map->find_next(); } } +//-------------------------------------------------------------------------- +// dump rule dependencies +//-------------------------------------------------------------------------- + using SvcMap = std::unordered_map>; static SvcMap get_dependencies() @@ -412,15 +475,19 @@ static SvcMap get_dependencies() void dump_rule_deps(const SnortConfig*) { SvcMap map = get_dependencies(); + JsonStream json(std::cout); for ( const auto& it : map ) { - dump_field("service", it.first, false); + json.open(); + json.put("service", it.first); + json.open_array("requires"); for ( const auto& s : it.second ) - dump_field("requires", s); + json.put(nullptr, s); - std::cout << std::endl; + json.close_array(); + json.close(); } } diff --git a/src/framework/parameter.cc b/src/framework/parameter.cc index 9dd6bd32a..e266be180 100644 --- a/src/framework/parameter.cc +++ b/src/framework/parameter.cc @@ -115,7 +115,11 @@ static int64_t get_int(const char* r) if ( !strncmp(r, "max53", 5) ) return 9007199254740992; } - return (int64_t)strtod(r, nullptr); + char* end = nullptr; + int64_t i = (int64_t)strtoll(r, &end, 0); + assert(!*end or *end == ':'); + + return i; } //-------------------------------------------------------------------------- @@ -580,20 +584,23 @@ num_tests[] = { true, valid_int, 1, "0:" }, { false, valid_int, 1, ":0" }, + { false, valid_int, 1.5, ":0" }, + { true, valid_int, -10, "-11:-9" }, { true, valid_int, 10, "9:11" }, { true, valid_int, 10, "0xA:11" }, { true, valid_interval, 0, nullptr }, - { true, valid_real, 0, nullptr }, - { true, valid_real, 0, "" }, - { true, valid_real, 0, "0.0" }, - { true, valid_real, 0, "0:" }, - { true, valid_real, 0, ":0" }, - { true, valid_real, 0, ":0.9" }, - { true, valid_real, 0, "-0.9:0.9" }, - { true, valid_real, 0, "-0.9:" }, + { true, valid_real, 0.0, nullptr }, + { true, valid_real, 0.0, "" }, + { true, valid_real, 0.0, "0.0" }, + { true, valid_real, 0.0, ":0" }, + + { true, valid_real, 0.1, "0:" }, + { true, valid_real, 0.1, ":0.9" }, + { true, valid_real, 0.1, "-0.9:0.9" }, + { true, valid_real, 0.1, "-0.9:" }, { false, valid_real, 1, "0.9" }, { true, valid_real, 1, "0.9:" }, diff --git a/src/helpers/CMakeLists.txt b/src/helpers/CMakeLists.txt index 754a9257b..af5367f0f 100644 --- a/src/helpers/CMakeLists.txt +++ b/src/helpers/CMakeLists.txt @@ -31,6 +31,8 @@ add_library (helpers OBJECT discovery_filter.cc discovery_filter.h flag_context.h + json_stream.cc + json_stream.h literal_search.cc markup.cc markup.h diff --git a/src/helpers/json_stream.cc b/src/helpers/json_stream.cc new file mode 100644 index 000000000..be554bdbf --- /dev/null +++ b/src/helpers/json_stream.cc @@ -0,0 +1,96 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// json_stream.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "json_stream.h" + +#include +#include + +void JsonStream::open(const char* key) +{ + split(); + + if ( key ) + out << std::quoted(key) << ": "; + + out << "{ "; + sep = false; + ++level; +} + +void JsonStream::close() +{ + out << " }"; + assert(level > 0); + + if ( --level == 0 ) + { + out << std::endl; + sep = false; + } +} + +void JsonStream::open_array(const char* key) +{ + split(); + out << std::quoted(key) << ": [ "; + sep = false; +} + +void JsonStream::close_array() +{ + out << " ]"; + sep = true; +} + +void JsonStream::put(const char* key, long val) +{ + split(); + + if ( key ) + out << std::quoted(key) << ": "; + + out << val; +} + +void JsonStream::put(const char* key, const std::string& val) +{ + if ( val.empty() ) + return; + + split(); + + if ( key ) + out << std::quoted(key) << ": "; + + out << std::quoted(val); +} + +void JsonStream::split() +{ + if ( sep ) + out << ", "; + else + sep = true; +} + diff --git a/src/helpers/json_stream.h b/src/helpers/json_stream.h new file mode 100644 index 000000000..7a4cf8656 --- /dev/null +++ b/src/helpers/json_stream.h @@ -0,0 +1,52 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// json_stream.h author Russ Combs + +#ifndef JSON_STREAM_H +#define JSON_STREAM_H + +// Simple output stream for outputting JSON data. + +#include + +class JsonStream +{ +public: + JsonStream(std::ostream& o) : out(o) { } + ~JsonStream() = default; + + void open(const char* key = nullptr); + void close(); + + void open_array(const char* key); + void close_array(); + + void put(const char* key, long val); + void put(const char* key, const std::string& val); + +private: + void split(); + +private: + std::ostream& out; + bool sep = false; + unsigned level = 0; +}; + +#endif + diff --git a/src/helpers/test/CMakeLists.txt b/src/helpers/test/CMakeLists.txt index 9b989cb6d..5255dd7e0 100644 --- a/src/helpers/test/CMakeLists.txt +++ b/src/helpers/test/CMakeLists.txt @@ -16,3 +16,9 @@ endif() add_catch_test( bitop_test ) +add_catch_test( json_stream_test + SOURCES + json_stream_test.cc + ../json_stream.cc +) + diff --git a/src/helpers/test/json_stream_test.cc b/src/helpers/test/json_stream_test.cc new file mode 100644 index 000000000..be07e6671 --- /dev/null +++ b/src/helpers/test/json_stream_test.cc @@ -0,0 +1,182 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// json_stream_test.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "catch/catch.hpp" + +#include "../json_stream.h" + +TEST_CASE("basic", "[json_stream]") +{ + std::ostringstream ss; + JsonStream js(ss); + + SECTION("empty body") + { + js.open(); + js.close(); + CHECK(ss.str() == "{ }\n"); + } + + SECTION("empty array") + { + js.open_array("a"); + js.close_array(); + const char* x = R"-("a": [ ])-"; + CHECK(ss.str() == x); + } + + SECTION("int") + { + js.put("i", 0); + const char* x = R"-("i": 0)-"; + CHECK(ss.str() == x); + } + + SECTION("string") + { + js.put("s", "yo"); + const char* x = R"-("s": "yo")-"; + CHECK(ss.str() == x); + } + + SECTION("empty string") + { + std::string mt; + js.put("s", mt); + CHECK(ss.str() == ""); + } + + SECTION("int item") + { + js.put(nullptr, 1); + CHECK(ss.str() == "1"); + } + + SECTION("string item") + { + js.put(nullptr, "it"); + const char* x = R"-("it")-"; + CHECK(ss.str() == x); + } + + SECTION("embedded quotes") + { + const char* s = R"-(content:"foo";)-"; + const char* x = R"-("content:\"foo\";")-"; + js.put(nullptr, s); + CHECK(ss.str() == x); + } + + SECTION("int list") + { + js.put("i", 2); + js.put("j", 3); + const char* x = R"-("i": 2, "j": 3)-"; + CHECK(ss.str() == x); + } + + SECTION("string list") + { + js.put("s", "alpha"); + js.put("t", "beta"); + js.put("u", "gamma"); + const char* x = R"-("s": "alpha", "t": "beta", "u": "gamma")-"; + CHECK(ss.str() == x); + } + + SECTION("array list") + { + js.open(); + js.open_array("m"); + js.close_array(); + js.open_array("n"); + js.close_array(); + js.close(); + const char* x = R"-({ "m": [ ], "n": [ ] })-" "\n"; + CHECK(ss.str() == x); + } + + SECTION("int array") + { + js.open(); + js.open_array("k"); + js.put(nullptr, 4); + js.put(nullptr, 5); + js.put(nullptr, 6); + js.close_array(); + js.close(); + const char* x = R"-({ "k": [ 4, 5, 6 ] })-" "\n"; + CHECK(ss.str() == x); + } + + SECTION("string array") + { + js.open(); + js.open_array("v"); + js.put(nullptr, "long"); + js.put(nullptr, "road"); + js.close_array(); + js.close(); + const char* x = R"-({ "v": [ "long", "road" ] })-" "\n"; + CHECK(ss.str() == x); + } + + SECTION("array list") + { + js.open(); + js.open_array("m"); + js.close_array(); + js.open_array("n"); + js.close_array(); + js.close(); + const char* x = R"-({ "m": [ ], "n": [ ] })-" "\n"; + CHECK(ss.str() == x); + } + + SECTION("int array int") + { + js.open(); + js.put("a", 7); + js.open_array("m"); + js.close_array(); + js.put("b", 8); + js.close(); + const char* x = R"-({ "a": 7, "m": [ ], "b": 8 })-" "\n"; + CHECK(ss.str() == x); + } + + SECTION("string array string") + { + js.open(); + js.put("c", "Snort"); + js.open_array("n"); + js.close_array(); + js.put("d", "++"); + js.close(); + const char* x = R"-({ "c": "Snort", "n": [ ], "d": "++" })-" "\n"; + CHECK(ss.str() == x); + } +} + diff --git a/src/managers/ips_manager.cc b/src/managers/ips_manager.cc index 5297a3f5e..321b9deaf 100644 --- a/src/managers/ips_manager.cc +++ b/src/managers/ips_manager.cc @@ -130,7 +130,7 @@ static bool set_arg( if ( p->is_wild_card() ) val = opt; - long n = (long)strtod(val, &end); + long n = (long)strtoll(val, &end, 0); if ( !*end ) v.set(n); diff --git a/src/parser/parse_so_rule.cc b/src/parser/parse_so_rule.cc index 9434ca651..9e0f501e8 100644 --- a/src/parser/parse_so_rule.cc +++ b/src/parser/parse_so_rule.cc @@ -162,14 +162,11 @@ bool SoRuleParser::parse_so_rule(const char* in, std::string& stub, std::string& } break; case 6: // in rule option - if ( c == '#' ) - { - state = 1; - next = 6; - drop = true; - break; - } - else if ( c == '/' ) + // FIXIT-L ideally we'd allow # comments within so rule stub options + // same as we do for non stubs but we should reuse the text rule parser + // instead of building this out. supporting them here is non-trivial + // (like state 5) because references can have # fragments. + if ( c == '/' ) { state = 2; next = 6; @@ -295,7 +292,8 @@ static const TestCase basic_tests[] = { "alert( sid:1; )", "alert( sid:1; )", true }, { "alert( sid:1 /*comment*/; )", "alert( sid:1 ; )", true }, - { "alert( sid:1 # comment\n; )", "alert( sid:1 ; )", true }, + // ideally below would be supported, but above works + { "alert( sid:1 # comment\n; )", "alert( sid:1 # comment ; )", true }, { "alert( sid:1; /*id:0;*/ )", "alert( sid:1; )", true }, { "alert tcp any any -> any any ( )",