From: Steve Chew (stechew) Date: Wed, 19 Oct 2022 16:20:12 +0000 (+0000) Subject: Pull request #3588: Add stateful signature evaluation X-Git-Tag: 3.1.45.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=59f9b5ff1588dfdb99562cd6b4754b4822ab37fd;p=thirdparty%2Fsnort3.git Pull request #3588: Add stateful signature evaluation Merge in SNORT/snort3 from ~OSHUMEIK/snort3:stateful_signature_evaluation to master Squashed commit of the following: commit 8477617f494ffebae8c95ad6456c7ce3b630b34b Author: Oleksii Shumeiko Date: Mon Apr 18 19:27:53 2022 +0300 detection: add stateful signature evaluation If an IPS option sets the cursor beyond the current buffer size, an evaluation state will be stored on the flow. Rule evaluation will resume later, when enough data from the buffer become available. Key updates/features: * buffers supported: pkt_data, file_data, js_data * a rule fired on the current packet doesn't create continuations * continuations are droppped on config reload * a few peg counters added * rule variables are transferred to the continuation * rule latency supported Continuation tracks stream source for the following buffers: pkt_data -- TCP payload data with respect to flow direction js_data -- JavaScript text combined within the same HTTP request/response file_data -- file's data within the same file (context) Now a leaf node can have children, which are flowbit setters moved to the very end. If an inspector sends PDU with data prepended from previous PDUs, Continuations will be dropped, because data chunks cannot be concatenated. Currently, http_inspect http2_inspect can present accumulated data in file_data and js_data buffers. --- diff --git a/doc/user/concepts.txt b/doc/user/concepts.txt index df3db3dc2..358b2540f 100644 --- a/doc/user/concepts.txt +++ b/doc/user/concepts.txt @@ -289,3 +289,24 @@ be skipped if possible. Note that this differs from Snort 2 which provided the fast_pattern:only option to designate such cases. This is one less thing for the rule writer to worry about. +===== Stateful Evaluation + +When data forms a kind of stream, e.g. contiguous byte flow (like a file +transferred over the network or byte sequence from TCP session packets), +the point of interest may be in a signature which spans across packets (its +parts lies in different data blocks). In this case, the stateful evaluation +becomes handy. + +If rule evaluation starts in a packet and the cursor position is moved beyond +the current packet boundary, then the evaluation gets paused and will resume +later when more data become available to finish the process. + +Stateful evaluation is supported for the following buffers: + + 1. pkt_data -- as a sequence of TCP session bytes with respect to their + direction (client-to-server, server-to-client) + + 2. js_data -- normalized JavaScript text from the same data transfer session + + 3. file_data -- the same file bytes, e.g. flows from different files do not + overlap diff --git a/src/detection/detect_trace.h b/src/detection/detect_trace.h index 4cf7a6510..09979365e 100644 --- a/src/detection/detect_trace.h +++ b/src/detection/detect_trace.h @@ -48,6 +48,7 @@ enum TRACE_PKT_DETECTION, TRACE_OPTION_TREE, TRACE_TAG, + TRACE_CONT, }; void clear_trace_cursor_info(); diff --git a/src/detection/detection_continuation.h b/src/detection/detection_continuation.h new file mode 100644 index 000000000..24d30c50d --- /dev/null +++ b/src/detection/detection_continuation.h @@ -0,0 +1,308 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// detection_continuation.h author Yehor Velykozhon + +#ifndef DETECTION_CONTINUATION_H +#define DETECTION_CONTINUATION_H + +#include "framework/cursor.h" +#include "framework/ips_option.h" +#include "ips_options/extract.h" +#include "latency/rule_latency.h" +#include "latency/rule_latency_state.h" +#include "main/thread_config.h" +#include "protocols/packet.h" +#include "trace/trace_api.h" +#include "utils/grouped_list.h" +#include "utils/stats.h" + +#include "detection_options.h" +#include "detect_trace.h" +#include "ips_context.h" +#include "rule_option_types.h" + +class Continuation +{ +public: + template + static inline void postpone(const Cursor&, + const detection_option_tree_node_t&, const detection_option_eval_data_t&); + + static inline void recall(dot_node_state_t&, const snort::Packet*); + + inline bool is_reloaded() const; + + inline void eval(snort::Packet&); + +private: + Continuation(int max = 1024) : states_cnt(0), states_cnt_max(max), + reload_id(snort::SnortConfig::get_thread_reload_id()) + { } + + template + inline void add(const Cursor&, + const detection_option_tree_node_t&, const detection_option_eval_data_t&); + + struct State + { + State() : data(), root(), selector(nullptr), node(nullptr), + waypoint(0), sid(0), packet_number(0), opt_parent(false) + { + for (uint8_t i = 0; i < NUM_IPS_OPTIONS_VARS; ++i) + byte_extract_vars[i] = 0; + } + + State(const detection_option_tree_node_t& n, const detection_option_eval_data_t& d, + snort::IpsOption* s, unsigned wp, uint64_t id, bool p) : data(d), + root(1, nullptr, d.otn, new RuleLatencyState[snort::ThreadConfig::get_instance_max()]()), + selector(s), node(const_cast(&n)), + waypoint(wp), sid(id), packet_number(d.p->context->packet_number), opt_parent(p) + { + for (uint8_t i = 0; i < NUM_IPS_OPTIONS_VARS; ++i) + snort::GetVarValueByIndex(&byte_extract_vars[i], i); + + root.children = &node; + } + + ~State() + { delete[] root.latency_state; } + + inline bool eval(snort::Packet&); + + detection_option_eval_data_t data; + detection_option_tree_root_t root; + snort::IpsOption* selector; + detection_option_tree_node_t* node; + unsigned waypoint; + uint64_t sid; + uint64_t packet_number; + uint32_t byte_extract_vars[NUM_IPS_OPTIONS_VARS]; + bool opt_parent; + }; + + using LState = snort::GroupedList; + + LState states; + unsigned states_cnt; + const unsigned states_cnt_max; + const unsigned reload_id; +}; + +template +void Continuation::postpone(const Cursor& cursor, + const detection_option_tree_node_t& node, const detection_option_eval_data_t& data) +{ + if (!cursor.awaiting_data()) + return; + + if (!data.p->has_paf_payload()) + return; + + assert(data.p->flow); + + auto cont = data.p->flow->ips_cont; + + if (!cont) + { + cont = data.p->flow->ips_cont = new Continuation(); + snort::pc.cont_flows++; + } + + cont->add(cursor, node, data); +} + +void Continuation::recall(dot_node_state_t& nst, + const snort::Packet* p) +{ + if (nst.last_check.context_num != nst.conts_num) + return; + + auto cnt = LState::erase_group((LState*&)nst.conts); + assert(nst.conts == nullptr); + assert(cnt); + + debug_logf(detection_trace, TRACE_CONT, nullptr, + "The path matched, dropping %u continuation(s)\n", cnt); + + assert(p); + assert(p->flow); + assert(p->flow->ips_cont); + + auto cont = p->flow->ips_cont; + + assert(cnt <= cont->states_cnt); + cont->states_cnt -= cnt; + snort::pc.cont_recalls += cnt; +} + +bool Continuation::is_reloaded() const +{ + return snort::SnortConfig::get_thread_reload_id() != reload_id; +} + +void Continuation::eval(snort::Packet& p) +{ + if (!p.has_paf_payload()) + return; + + debug_logf(detection_trace, TRACE_CONT, nullptr, + "Processing %u continuation(s)\n", states_cnt); + + if (states_cnt > snort::pc.cont_max_num) + snort::pc.cont_max_num = states_cnt; + + auto i = states.get_next(); + + while (i != &states) + { + auto st = i; + i = i->get_next(); + + if ((**st).eval(p)) + { + assert(0 < states_cnt); + assert(st != &states); + --states_cnt; + st->leave_group(); + delete st; + } + } +} + +bool Continuation::State::eval(snort::Packet& p) +{ + Cursor cursor(&p); + + // Create local object to follow latency workflow + RuleLatency::Context rule_latency_ctx(root, &p); + + if (RuleLatency::suspended()) + return true; + + if (packet_number == p.context->packet_number) + return false; + + if (selector) + selector->eval(cursor, &p); + + if (sid != cursor.id()) + return false; + + if (cursor.is_buffer_accumulated()) + { + debug_logf(detection_trace, TRACE_CONT, nullptr, + "Continuation dropped due to accumulated data in '%s'\n", cursor.get_name()); + + return true; + } + + cursor.set_pos(waypoint); + + if (cursor.awaiting_data(true) or cursor.size() == 0) + { + waypoint = cursor.get_next_pos(); + debug_logf(detection_trace, TRACE_CONT, data.p, + "Continuation postponed, %u bytes to go\n", waypoint); + return false; + } + + assert(cursor.get_name()); + debug_logf(detection_trace, TRACE_CONT, data.p, + "Cursor reached the position, evaluating sub-tree with " + "current buffer '%s'\n", cursor.get_name()); + + data.pmd = nullptr; + data.p = &p; + + int result = 0; + snort::pc.cont_evals++; + + for (uint8_t i = 0; i < NUM_IPS_OPTIONS_VARS; ++i) + snort::SetVarValueByIndex(byte_extract_vars[i], i); + + const detection_option_tree_node_t* node = root.children[0]; + + if (opt_parent) + { + for (int i = 0; i < node->num_children; ++i) + result += detection_option_node_evaluate(node->children[i], data, cursor); + } + else + { + result = detection_option_node_evaluate(node, data, cursor); + } + + clear_trace_cursor_info(); + + if (result) + snort::pc.cont_matches++; + else + snort::pc.cont_mismatches++; + + return true; +} + +template +void Continuation::add(const Cursor& cursor, + const detection_option_tree_node_t& node, const detection_option_eval_data_t& data) +{ + auto selector = data.buf_selector; + auto pos = cursor.get_next_pos(); + auto sid = cursor.id(); + auto nst = node.state + snort::get_instance_id(); + assert(nst); + + if (nst->last_check.context_num != nst->conts_num) + { + nst->conts_num = nst->last_check.context_num; + nst->conts = nullptr; + } + + if (states_cnt < states_cnt_max) + { + ++states_cnt; + new LState(states, (LState*&)nst->conts, node, data, selector, pos, sid, opt_parent); + } + else + { + debug_logf(detection_trace, TRACE_CONT, data.p, + "Max size reached (%u), rejecting the oldest continuation\n", + states_cnt_max); + + auto st = states.get_next(); + assert(st != &states); + + if ((**st).packet_number == data.p->context->packet_number) + st->leave_group(); + delete st; + + new LState(states, (LState*&)nst->conts, node, data, selector, pos, sid, opt_parent); + } + + snort::pc.cont_creations++; + + assert(cursor.get_name()); + assert(!selector || selector->get_name()); + debug_logf(detection_trace, TRACE_CONT, data.p, + "Adding a continuation: " + "current buffer '%s', buffer selector '%s'; %u more bytes needed\n", + cursor.get_name(), selector ? selector->get_name() : "N/A", pos); +} + +#endif diff --git a/src/detection/detection_engine.cc b/src/detection/detection_engine.cc index eeea78164..f8c3f70eb 100644 --- a/src/detection/detection_engine.cc +++ b/src/detection/detection_engine.cc @@ -105,6 +105,7 @@ DetectionEngine::DetectionEngine() context = Analyzer::get_switcher()->interrupt(); context->file_data = DataPointer(nullptr, 0); + context->file_data_id = 0; reset(); } @@ -299,11 +300,34 @@ DataBuffer& DetectionEngine::get_alt_buffer(Packet* p) } void DetectionEngine::set_file_data(const DataPointer& dp) -{ Analyzer::get_switcher()->get_context()->file_data = dp; } +{ + auto c = Analyzer::get_switcher()->get_context(); + c->file_data = dp; + c->file_data_id = 0; + c->file_data_drop_sse = false; + c->file_data_no_sse = false; +} + +void DetectionEngine::set_file_data(const DataPointer& dp, uint64_t id, bool is_accum, bool no_flow) +{ + auto c = Analyzer::get_switcher()->get_context(); + c->file_data = dp; + c->file_data_id = id; + c->file_data_drop_sse = is_accum; + c->file_data_no_sse = no_flow; +} -DataPointer& DetectionEngine::get_file_data(IpsContext* c) +const DataPointer& DetectionEngine::get_file_data(const IpsContext* c) { return c->file_data; } +const DataPointer& DetectionEngine::get_file_data(const IpsContext* c, uint64_t& id, bool& drop_sse, bool& no_sse) +{ + id = c->file_data_id; + drop_sse = c->file_data_drop_sse; + no_sse = c->file_data_no_sse; + return c->file_data; +} + void DetectionEngine::set_data(unsigned id, IpsContextData* p) { Analyzer::get_switcher()->get_context()->set_context_data(id, p); } diff --git a/src/detection/detection_engine.h b/src/detection/detection_engine.h index 5b62a0fb0..44533aa70 100644 --- a/src/detection/detection_engine.h +++ b/src/detection/detection_engine.h @@ -69,7 +69,9 @@ public: static Packet* get_encode_packet(); static void set_file_data(const DataPointer& dp); - static DataPointer& get_file_data(IpsContext*); + static void set_file_data(const DataPointer& dp, uint64_t id, bool is_accum, bool no_flow); + static const DataPointer& get_file_data(const IpsContext*); + static const DataPointer& get_file_data(const IpsContext*, uint64_t& id, bool& drop_sse, bool& no_sse); static uint8_t* get_buffer(unsigned& max); static struct DataBuffer& get_alt_buffer(Packet*); @@ -127,6 +129,12 @@ static inline void set_file_data(const uint8_t* p, unsigned n) DetectionEngine::set_file_data(dp); } +static inline void set_file_data(const uint8_t* p, unsigned n, uint64_t id, bool is_accum = false, bool no_flow = false) +{ + DataPointer dp { p, n }; + DetectionEngine::set_file_data(dp, id, is_accum, no_flow); +} + static inline void clear_file_data() { set_file_data(nullptr, 0); } diff --git a/src/detection/detection_module.cc b/src/detection/detection_module.cc index 8bf8273e4..2d5084277 100644 --- a/src/detection/detection_module.cc +++ b/src/detection/detection_module.cc @@ -49,6 +49,7 @@ static const TraceOption detection_trace_options[] = { "pkt_detect", TRACE_PKT_DETECTION, "enable packet detection trace logging" }, { "opt_tree", TRACE_OPTION_TREE, "enable tree option trace logging" }, { "tag", TRACE_TAG, "enable tag trace logging" }, + { "cont", TRACE_CONT, "enable rule continuation trace logging" }, { nullptr, 0, nullptr } }; #endif diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index ab2fb627a..acc5bed6f 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -54,6 +54,7 @@ #include "utils/util.h" #include "utils/util_cstring.h" +#include "detection_continuation.h" #include "detection_engine.h" #include "detection_module.h" #include "detection_util.h" @@ -348,26 +349,17 @@ void* add_detection_option_tree(SnortConfig* sc, detection_option_tree_node_t* o } int detection_option_node_evaluate( - detection_option_tree_node_t* node, detection_option_eval_data_t& eval_data, + const detection_option_tree_node_t* node, detection_option_eval_data_t& eval_data, const Cursor& orig_cursor) { assert(node and eval_data.p); + node_eval_trace(node, orig_cursor, eval_data.p); + auto& state = node->state[get_instance_id()]; RuleContext profile(state); - int result = 0; - int rval; - char tmp_noalert_flag = 0; - Cursor cursor = orig_cursor; - bool continue_loop = true; - bool flowbits_setoperation = false; - int loop_count = 0; - uint32_t tmp_byte_extract_vars[NUM_IPS_OPTIONS_VARS]; uint64_t cur_eval_context_num = eval_data.p->context->context_num; - - node_eval_trace(node, cursor, eval_data.p); - auto p = eval_data.p; // see if evaluated it before ... @@ -410,6 +402,16 @@ int detection_option_node_evaluate( content_last = pmd->last_check + get_instance_id(); } + bool continue_loop = true; + int loop_count = 0; + + char tmp_noalert_flag = 0; + int result = 0; + uint32_t tmp_byte_extract_vars[NUM_IPS_OPTIONS_VARS]; + IpsOption* buf_selector = eval_data.buf_selector; + Cursor cursor = orig_cursor; + int rval; + // No, haven't evaluated this one before... Check it. do { @@ -511,20 +513,20 @@ int detection_option_node_evaluate( case RULE_OPTION_TYPE_FLOWBIT: if ( node->evaluate ) { - flowbits_setoperation = flowbits_setter(node->option_data); - - if ( flowbits_setoperation ) - // set to match so we don't bail early - rval = (int)IpsOption::MATCH; - - else - rval = node->evaluate(node->option_data, cursor, eval_data.p); + rval = node->evaluate(node->option_data, cursor, eval_data.p); + assert((flowbits_setter(node->option_data) and rval == (int)IpsOption::MATCH) + or !flowbits_setter(node->option_data)); } break; default: if ( node->evaluate ) + { + IpsOption* opt = (IpsOption*)node->option_data; + if ( opt->is_buffer_setter() ) + buf_selector = opt; rval = node->evaluate(node->option_data, cursor, p); + } break; } @@ -532,6 +534,11 @@ int detection_option_node_evaluate( { debug_log(detection_trace, TRACE_RULE_EVAL, p, "no match\n"); state.last_check.result = result; + // See if the option has failed due to incomplete input or its failed evaluation + if (orig_cursor.awaiting_data()) + Continuation::postpone(orig_cursor, *node, eval_data); + else + Continuation::postpone(cursor, *node, eval_data); return result; } else if ( rval == (int)IpsOption::FAILED_BIT ) @@ -644,6 +651,7 @@ int detection_option_node_evaluate( continue; } + eval_data.buf_selector = buf_selector; child_state->result = detection_option_node_evaluate( node->children[i], eval_data, cursor); @@ -668,10 +676,14 @@ int detection_option_node_evaluate( // the children so don't need to reeval this content/pcre rule // option at a new offset. // Else, reset the DOE ptr to last eval for offset/depth, - // distance/within adjustments for this same content/pcre - // rule option + // distance/within adjustments for this same content/pcre rule option. + // If the node and its sub-tree propagate MATCH back, + // then all its continuations are recalled. if ( result == node->num_children ) + { continue_loop = false; + Continuation::recall(state, p); + } // Don't need to reset since it's only checked after we've gone // through the loop at least once and the result will have @@ -706,15 +718,6 @@ int detection_option_node_evaluate( } while ( continue_loop ); - if ( flowbits_setoperation && result == (int)IpsOption::MATCH ) - { - // Do any setting/clearing/resetting/toggling of flowbits here - // given that other rule options matched - rval = node->evaluate(node->option_data, cursor, p); - if ( rval != (int)IpsOption::MATCH ) - result = rval; - } - if ( eval_data.flowbit_failed ) { // something deeper in the tree failed a flowbit test, we may need to diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h index 503de9c8c..556fa2bd7 100644 --- a/src/detection/detection_options.h +++ b/src/detection/detection_options.h @@ -40,6 +40,7 @@ namespace snort { class HashNode; +class IpsOption; class XHash; struct Packet; struct SnortConfig; @@ -61,6 +62,8 @@ struct dot_node_state_t char result; char flowbit_failed; } last_check; + void* conts; + uint64_t conts_num; // FIXIT-L perf profiler stuff should be factored of the node state struct hr_duration elapsed; @@ -86,41 +89,64 @@ struct dot_node_state_t } }; -struct detection_option_tree_node_t +struct detection_option_tree_node_t; + +struct detection_option_tree_bud_t { - eval_func_t evaluate; + int relative_children; + int num_children; detection_option_tree_node_t** children; + const struct OptTreeNode* otn; + + detection_option_tree_bud_t() + : relative_children(0), num_children(0), children(nullptr), otn(nullptr) {} + + detection_option_tree_bud_t(int num, detection_option_tree_node_t** d_otn, const OptTreeNode* r_otn) + : relative_children(0), num_children(num), children(d_otn), otn(r_otn) {} +}; + +struct detection_option_tree_node_t : public detection_option_tree_bud_t +{ + eval_func_t evaluate; void* option_data; dot_node_state_t* state; - struct OptTreeNode* otn; int is_relative; - int num_children; - int relative_children; option_type_t option_type; }; -struct detection_option_tree_root_t +struct detection_option_tree_root_t : public detection_option_tree_bud_t { - int num_children; - detection_option_tree_node_t** children; RuleLatencyState* latency_state; - struct OptTreeNode* otn; // first rule in tree + detection_option_tree_root_t() + : detection_option_tree_bud_t(), latency_state(nullptr) {} + + detection_option_tree_root_t(int num, detection_option_tree_node_t** d_otn, const OptTreeNode* r_otn, + RuleLatencyState* lat) + : detection_option_tree_bud_t(num, d_otn, r_otn), latency_state(lat) {} }; struct detection_option_eval_data_t { - detection_option_eval_data_t() = delete; - - detection_option_eval_data_t(snort::Packet* p) : - pmd(nullptr), p(p), leaf_reached(0), flowbit_failed(0), flowbit_noalert(0) - { } - - void* pmd; + const void* pmd; snort::Packet* p; + snort::IpsOption* buf_selector; + const struct OptTreeNode* otn; // first rule in current processed tree char leaf_reached; char flowbit_failed; char flowbit_noalert; + + detection_option_eval_data_t() + : pmd(nullptr), p(nullptr), buf_selector(nullptr), otn(nullptr) + , leaf_reached(0), flowbit_failed(0), flowbit_noalert(0) {} + + detection_option_eval_data_t(snort::Packet* packet, const OptTreeNode* otn, + const void* match_data = nullptr) : pmd(match_data), p(packet), buf_selector(nullptr) + , otn(otn), leaf_reached(0), flowbit_failed(0), flowbit_noalert(0) {} + + detection_option_eval_data_t(const detection_option_eval_data_t& m) + : pmd(m.pmd), p(m.p), buf_selector(m.buf_selector), otn(m.otn) + , leaf_reached(m.leaf_reached), flowbit_failed(m.flowbit_failed), flowbit_noalert(m.flowbit_noalert) {} }; // return existing data or add given and return nullptr @@ -128,7 +154,7 @@ void* add_detection_option(struct snort::SnortConfig*, option_type_t, void*); void* add_detection_option_tree(struct snort::SnortConfig*, detection_option_tree_node_t*); int detection_option_node_evaluate( - detection_option_tree_node_t*, detection_option_eval_data_t&, const class Cursor&); + const detection_option_tree_node_t*, detection_option_eval_data_t&, const class Cursor&); void print_option_tree(detection_option_tree_node_t*, int level); void detection_option_tree_update_otn_stats(snort::XHash*); diff --git a/src/detection/dev_notes.txt b/src/detection/dev_notes.txt index aeaba6692..186f2053c 100644 --- a/src/detection/dev_notes.txt +++ b/src/detection/dev_notes.txt @@ -199,3 +199,67 @@ When packets arrive: } } +*Stateful Signature Evaluation* + +Detection module processes data in block mode. Some features are available to +extend limits of the block mode: large PDU with accumulated data (rebuilt +packets), flowbits, and stateful signature evaluation (SSE). +For data which forms a contiguous flow, a rule evaluation started in one +packet may end up in subsequent packets. + +From the user perspective, rule options are evaluated in left-to-right order, +doing their calculations, checks, setting cursor position, switching buffers, +etc. If at some point the cursor position is set beyond the current buffer +boundary, when the rule's evaluation context is stored on the flow and the +rule fails for the current packet. Later, when more data arrives, Detection +module checks if the flow has suspended contexts and resumes their evaluation +after rule group selection and fast-pattern search are done but before other +rules evaluated. + +The Continuation concept represents a suspended evaluation: + +1. IPS options are evaluated in a known order, where they form pairs when + the 1st option passes the evaluation context to the next option. + +2. The Continuation is merely the context (or a state) with a few things + attached. So, it is somewhere between options, rather than in an option. + +3. The statement above lets us deal with a single evaluation pass, which is + independent of what goes before and after it. (Avoiding multiple children + nodes). + +4. Evaluation flow goes forward and backwards on this pass. The pass could be + attempted several times, depending on presence of retry IPS options. + +5. On each attempt, the following could happen: + +Forward: + +[options="header"] +|=============================================================================== +| 1st option | 2nd option | Continuation is ... +| NO_MATCH | not evaluated | created and attached to the parent option (if cursor is awaiting data) +| MATCH | NO_MATCH | created and attached to the child option (if original cursor was awaiting data) +| MATCH | MATCH | not created (for this pass, but a subsequent pass can have its own continuation) +|=============================================================================== + +At this point, it doesn't matter what happens after in the sub-tree. +If the 1st and the 2nd options matched, that means a continuation is not +needed here. + +Backward: + +[options="header"] +|=============================================================================== +| option | children | Continuations spawned by the option ... +| (matched) | no match or partially matched | remain attached to the flow +| (matched) | all matched | are recalled and erased from the flow +|=============================================================================== + +So, if some branches in the sub-tree failed to match, then continuations +created at this node will give another try to these branches (later when more +data become available). If sub-tree fully matched, then continuations are not +needed (since the rule has fired) and will be recalled. + +Pending continuations from the flow are picked up and updated/evaluated with +respect to the buffer's source (e.g. flow direction, file context, etc.) diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 16825b8b0..128eb8f5d 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -79,7 +79,7 @@ static void print_fp_info(const char*, const OptTreeNode*, const PatternMatchDat static OptTreeNode* fixup_tree( detection_option_tree_node_t* dot, bool branched, unsigned contents) { - if ( dot->num_children == 0 ) + if ( dot->num_children == 0 or dot->option_type == RULE_OPTION_TYPE_LEAF_NODE ) { if ( !branched and contents ) return (OptTreeNode*)dot->option_data; @@ -153,28 +153,28 @@ static bool new_sig(int num_children, detection_option_tree_node_t** nodes, OptT static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseType mpse_type) { - detection_option_tree_node_t* node = nullptr, * child; - bool need_leaf = false; - if (!existing_tree) return -1; if (!*existing_tree) *existing_tree = new_root(otn); - detection_option_tree_root_t* root = (detection_option_tree_root_t*)*existing_tree; + detection_option_tree_root_t* const root = (detection_option_tree_root_t*)*existing_tree; + detection_option_tree_bud_t* bud = root; + bool need_leaf = false; - if (!root->children) + if (!bud->children) { - root->num_children++; - root->children = (detection_option_tree_node_t**) - snort_calloc(root->num_children, sizeof(detection_option_tree_node_t*)); + bud->num_children++; + bud->children = (detection_option_tree_node_t**) + snort_calloc(bud->num_children, sizeof(detection_option_tree_node_t*)); need_leaf = true; } int i = 0; - child = root->children[i]; + detection_option_tree_node_t* child = bud->children[i]; OptFpList* opt_fp = otn->opt_func; + std::list fbss; /* Build out sub-nodes for each option in the OTN fp list */ while (opt_fp) @@ -197,61 +197,41 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseTyp continue; } + // A flowbit setter will be evaluated last. + if ( is_flowbit_setter(opt_fp) ) + { + fbss.push_back(opt_fp); + opt_fp = opt_fp->next; + continue; + } + if (!child) { /* No children at this node */ child = new_node(opt_fp->type, option_data); child->evaluate = opt_fp->OptTestFunc; - if (!node) - root->children[i] = child; - else - node->children[i] = child; + bud->children[i] = child; child->num_children++; child->children = (detection_option_tree_node_t**) snort_calloc(child->num_children, sizeof(detection_option_tree_node_t*)); child->is_relative = opt_fp->isRelative; - if (node && child->is_relative) - node->relative_children++; + if (child->is_relative) + bud->relative_children++; need_leaf = true; } else { - bool found_child_match = false; + bool found_child_match = child->option_data == option_data; - if (child->option_data != option_data) - { - if (!node) - { - for (i=1; inum_children; i++) - { - child = root->children[i]; - if (child->option_data == option_data) - { - found_child_match = true; - break; - } - } - } - else - { - for (i=1; inum_children; i++) - { - child = node->children[i]; - if (child->option_data == option_data) - { - found_child_match = true; - break; - } - } - } - } - else + for (i = 1; !found_child_match && i < bud->num_children; i++) { - found_child_match = true; + child = bud->children[i]; + if (child->option_data == option_data) + found_child_match = true; } if ( !found_child_match ) @@ -265,88 +245,82 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseTyp snort_calloc(child->num_children, sizeof(child->children)); child->is_relative = opt_fp->isRelative; - if (!node) - { - root->num_children++; - tmp_children = (detection_option_tree_node_t**) - snort_calloc(root->num_children, sizeof(tmp_children)); - memcpy(tmp_children, root->children, - sizeof(detection_option_tree_node_t*) * (root->num_children-1)); - - snort_free(root->children); - root->children = tmp_children; - root->children[root->num_children-1] = child; - } - else - { - node->num_children++; - tmp_children = (detection_option_tree_node_t**) - snort_calloc(node->num_children, sizeof(detection_option_tree_node_t*)); - memcpy(tmp_children, node->children, - sizeof(detection_option_tree_node_t*) * (node->num_children-1)); - - snort_free(node->children); - node->children = tmp_children; - node->children[node->num_children-1] = child; - if (child->is_relative) - node->relative_children++; - } + bud->num_children++; + tmp_children = (detection_option_tree_node_t**) + snort_calloc(bud->num_children, sizeof(detection_option_tree_node_t*)); + memcpy(tmp_children, bud->children, + sizeof(detection_option_tree_node_t*) * (bud->num_children-1)); + + snort_free(bud->children); + bud->children = tmp_children; + bud->children[bud->num_children-1] = child; + + if (child->is_relative) + bud->relative_children++; + need_leaf = true; } } - node = child; + bud = child; i=0; - child = node->children[i]; + child = bud->children[i]; opt_fp = opt_fp->next; } // don't add a new leaf node unless we branched higher in the tree or this // is a different sig ( eg alert ip ( sid:1; ) vs alert tcp ( sid:2; ) ) // note: same sig different policy branches at rtn (this is for same policy) - - if ( !need_leaf ) - { - if ( node ) - need_leaf = new_sig(node->num_children, node->children, otn); - else - need_leaf = new_sig(root->num_children, root->children, otn); - } - - if ( !need_leaf ) + if ( !need_leaf and !new_sig(bud->num_children, bud->children, otn) ) return 0; /* Append a leaf node that has option data of the SigInfo/otn pointer */ child = new_node(RULE_OPTION_TYPE_LEAF_NODE, otn); - if (!node) + if (bud->children[0]) { - if (root->children[0]) - { - detection_option_tree_node_t** tmp_children; - root->num_children++; - tmp_children = (detection_option_tree_node_t**) - snort_calloc(root->num_children, sizeof(detection_option_tree_node_t*)); - memcpy(tmp_children, root->children, - sizeof(detection_option_tree_node_t*) * (root->num_children-1)); - snort_free(root->children); - root->children = tmp_children; - } - root->children[root->num_children-1] = child; + detection_option_tree_node_t** tmp_children; + bud->num_children++; + tmp_children = (detection_option_tree_node_t**) + snort_calloc(bud->num_children, sizeof(detection_option_tree_node_t*)); + memcpy(tmp_children, bud->children, + sizeof(detection_option_tree_node_t*) * (bud->num_children - 1)); + snort_free(bud->children); + bud->children = tmp_children; } - else + bud->children[bud->num_children - 1] = child; + + // Build after-leaf nodes with flowbit setters. + auto fbss_num = fbss.size(); + + if (!fbss_num) + return 0; + + bud = child; + i = 0; + + bud->num_children = fbss_num; + bud->children = (detection_option_tree_node_t**) + snort_calloc(bud->num_children, sizeof(detection_option_tree_node_t*)); + + for (auto fbs : fbss) { - if (node->children[0]) + // Only flowbit setters should get here. + if (fbs->type == RULE_OPTION_TYPE_LEAF_NODE + or is_fast_pattern_only(otn, fbs, mpse_type) + or !is_flowbit_setter(fbs) ) { - detection_option_tree_node_t** tmp_children; - node->num_children++; - tmp_children = (detection_option_tree_node_t**) - snort_calloc(node->num_children, sizeof(detection_option_tree_node_t*)); - memcpy(tmp_children, node->children, - sizeof(detection_option_tree_node_t*) * (node->num_children-1)); - snort_free(node->children); - node->children = tmp_children; + assert(false); + bud->num_children--; + continue; } - node->children[node->num_children-1] = child; + + void* option_data = fbs->ips_opt; + child = new_node(fbs->type, option_data); + child->evaluate = fbs->OptTestFunc; + child->is_relative = fbs->isRelative; + bud->children[i++] = child; + + assert(child->is_relative == false); } return 0; diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index b3c0e0493..eb9a5a88a 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -69,10 +69,11 @@ #include "context_switcher.h" #include "detect.h" #include "detect_trace.h" -#include "detection_util.h" +#include "detection_continuation.h" #include "detection_engine.h" #include "detection_module.h" #include "detection_options.h" +#include "detection_util.h" #include "fp_config.h" #include "fp_create.h" #include "fp_utils.h" @@ -341,7 +342,7 @@ static void detection_option_tree_evaluate(detection_option_tree_root_t* root, { assert(root); - RuleLatency::Context rule_latency_ctx(root, eval_data.p); + RuleLatency::Context rule_latency_ctx(*root, eval_data.p); if ( RuleLatency::suspended() ) return; @@ -361,11 +362,9 @@ static void rule_tree_match( IpsContext* context, void* user, void* tree, int index, void* neg_list) { PMX* pmx = (PMX*)user; + Packet* pkt = context->packet; - detection_option_eval_data_t eval_data(context->packet); - eval_data.pmd = pmx->pmd; - - print_pattern(pmx->pmd, eval_data.p); + print_pattern(pmx->pmd, pkt); { /* NOTE: The otn will be the first one in the match state. If there are @@ -381,17 +380,18 @@ static void rule_tree_match( PmdLastCheck* last_check = neg_pmx->pmd->last_check + get_instance_id(); - last_check->ts.tv_sec = eval_data.p->pkth->ts.tv_sec; - last_check->ts.tv_usec = eval_data.p->pkth->ts.tv_usec; + last_check->ts.tv_sec = pkt->pkth->ts.tv_sec; + last_check->ts.tv_usec = pkt->pkth->ts.tv_usec; last_check->run_num = get_run_num(); last_check->context_num = context->context_num; - last_check->rebuild_flag = (eval_data.p->packet_flags & PKT_REBUILT_STREAM); + last_check->rebuild_flag = (pkt->packet_flags & PKT_REBUILT_STREAM); } if ( !tree ) return; detection_option_tree_root_t* root = (detection_option_tree_root_t*)tree; + detection_option_eval_data_t eval_data(pkt, root->otn, pmx->pmd); detection_option_tree_evaluate(root, eval_data); @@ -399,47 +399,47 @@ static void rule_tree_match( pmqs.qualified_events++; else pmqs.non_qualified_events++; - } - if (eval_data.flowbit_failed) - return; + if (eval_data.flowbit_failed) + return; + } /* If this is for an IP rule set, evaluate the rules from * the inner IP offset as well */ - if (eval_data.p->packet_flags & PKT_IP_RULE) + if (pkt->packet_flags & PKT_IP_RULE) { - ip::IpApi tmp_api = eval_data.p->ptrs.ip_api; - int8_t curr_layer = eval_data.p->num_layers - 1; + ip::IpApi tmp_api = pkt->ptrs.ip_api; + int8_t curr_layer = pkt->num_layers - 1; - if (layer::set_inner_ip_api(eval_data.p, - eval_data.p->ptrs.ip_api, + if (layer::set_inner_ip_api(pkt, + pkt->ptrs.ip_api, curr_layer) && - (eval_data.p->ptrs.ip_api != tmp_api)) + (pkt->ptrs.ip_api != tmp_api)) { - const uint8_t* tmp_data = eval_data.p->data; - uint16_t tmp_dsize = eval_data.p->dsize; + const uint8_t* tmp_data = pkt->data; + uint16_t tmp_dsize = pkt->dsize; /* clear so we don't keep recursing */ - eval_data.p->packet_flags &= ~PKT_IP_RULE; - eval_data.p->packet_flags |= PKT_IP_RULE_2ND; + pkt->packet_flags &= ~PKT_IP_RULE; + pkt->packet_flags |= PKT_IP_RULE_2ND; do { - eval_data.p->data = eval_data.p->ptrs.ip_api.ip_data(); - eval_data.p->dsize = eval_data.p->ptrs.ip_api.pay_len(); + pkt->data = pkt->ptrs.ip_api.ip_data(); + pkt->dsize = pkt->ptrs.ip_api.pay_len(); /* Recurse, and evaluate with the inner IP */ rule_tree_match(context, user, tree, index, nullptr); } - while (layer::set_inner_ip_api(eval_data.p, - eval_data.p->ptrs.ip_api, curr_layer) && (eval_data.p->ptrs.ip_api != tmp_api)); + while (layer::set_inner_ip_api(pkt, + pkt->ptrs.ip_api, curr_layer) && (pkt->ptrs.ip_api != tmp_api)); /* cleanup restore original data & dsize */ - eval_data.p->packet_flags &= ~PKT_IP_RULE_2ND; - eval_data.p->packet_flags |= PKT_IP_RULE; + pkt->packet_flags &= ~PKT_IP_RULE_2ND; + pkt->packet_flags |= PKT_IP_RULE; - eval_data.p->data = tmp_data; - eval_data.p->dsize = tmp_dsize; + pkt->data = tmp_data; + pkt->dsize = tmp_dsize; } } } @@ -1022,13 +1022,12 @@ static inline void eval_nfp( if ( fp->get_debug_print_nc_rules() ) LogMessage("NC-testing %u rules\n", port_group->nfp_rule_count); - detection_option_eval_data_t eval_data(p); + detection_option_tree_root_t* root = (detection_option_tree_root_t*)port_group->nfp_tree; + detection_option_eval_data_t eval_data(p, root->otn); debug_log(detection_trace, TRACE_RULE_EVAL, p, "Testing non-content rules\n"); - detection_option_tree_root_t* root = (detection_option_tree_root_t*)port_group->nfp_tree; - detection_option_tree_evaluate(root, eval_data); if (eval_data.leaf_reached) @@ -1300,6 +1299,18 @@ void fp_complete(Packet* p, bool search) } { Profile rule_profile(rulePerfStats); + + if (p->flow && p->flow->ips_cont) + { + if (!p->flow->ips_cont->is_reloaded()) + p->flow->ips_cont->eval(*p); + else + { + delete p->flow->ips_cont; + p->flow->ips_cont = nullptr; + } + } + stash->process(c); print_pkt_info(p, "non-fast-patterns"); fpEvalPacket(p, FPTask::NON_FP); diff --git a/src/detection/fp_utils.cc b/src/detection/fp_utils.cc index a94523918..9ec2644ee 100644 --- a/src/detection/fp_utils.cc +++ b/src/detection/fp_utils.cc @@ -36,6 +36,7 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "hash/ghash.h" +#include "ips_options/ips_flowbits.h" #include "log/messages.h" #include "main/snort_config.h" #include "parser/parse_conf.h" @@ -650,6 +651,12 @@ bool is_fast_pattern_only(const OptTreeNode* otn, const OptFpList* ofp, Mpse::Mp return false; } +bool is_flowbit_setter(const OptFpList* ofp) +{ + return ofp->type == RULE_OPTION_TYPE_FLOWBIT + and flowbits_setter(ofp->ips_opt); +} + //-------------------------------------------------------------------------- // mpse compile threads //-------------------------------------------------------------------------- diff --git a/src/detection/fp_utils.h b/src/detection/fp_utils.h index f2d870046..2d902a1a5 100644 --- a/src/detection/fp_utils.h +++ b/src/detection/fp_utils.h @@ -37,6 +37,7 @@ struct PatternMatchData* get_pmd(OptFpList*, SnortProtocolId, snort::RuleDirecti bool make_fast_pattern_only(const OptFpList*, const PatternMatchData*); bool is_fast_pattern_only(const OptTreeNode*, const OptFpList*, snort::Mpse::MpseType); +bool is_flowbit_setter(const OptFpList*); PatternMatcher::Type get_pm_type(const std::string& buf); diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h index 937482e4e..8d35f89a4 100644 --- a/src/detection/ips_context.h +++ b/src/detection/ips_context.h @@ -156,6 +156,9 @@ public: SF_EVENTQ* equeue; DataPointer file_data = DataPointer(nullptr, 0); + uint64_t file_data_id = 0; + bool file_data_drop_sse = false; + bool file_data_no_sse = false; DataBuffer alt_data = {}; unsigned file_pos = 0; bool file_type_process = false; diff --git a/src/flow/flow.cc b/src/flow/flow.cc index 3e5342ae5..1e27dacb0 100644 --- a/src/flow/flow.cc +++ b/src/flow/flow.cc @@ -24,6 +24,7 @@ #include "flow.h" #include "detection/context_switcher.h" +#include "detection/detection_continuation.h" #include "detection/detection_engine.h" #include "flow/flow_key.h" #include "flow/ha.h" @@ -121,6 +122,9 @@ void Flow::term() stash = nullptr; } + delete ips_cont; + ips_cont = nullptr; + service = nullptr; } @@ -219,6 +223,9 @@ void Flow::reset(bool do_cleanup) deferred_trust.clear(); + delete ips_cont; + ips_cont = nullptr; + constexpr size_t offset = offsetof(Flow, context_chain); // FIXIT-L need a struct to zero here to make future proof memset((uint8_t*)this+offset, 0, sizeof(Flow)-offset); diff --git a/src/flow/flow.h b/src/flow/flow.h index f057c27ee..578aead7c 100644 --- a/src/flow/flow.h +++ b/src/flow/flow.h @@ -102,6 +102,7 @@ #define STREAM_STATE_BLOCK_PENDING 0x0400 #define STREAM_STATE_RELEASING 0x0800 +class Continuation; class BitOp; class Session; @@ -427,6 +428,7 @@ public: // FIXIT-M privatize if possible Session* session; Inspector* ssn_client; Inspector* ssn_server; + Continuation* ips_cont; long last_data_seen; Layer mpls_client, mpls_server; diff --git a/src/framework/base_api.h b/src/framework/base_api.h index 1345d9de7..9a22b5e77 100644 --- a/src/framework/base_api.h +++ b/src/framework/base_api.h @@ -29,7 +29,7 @@ // this is the current version of the base api // must be prefixed to subtype version -#define BASE_API_VERSION 15 +#define BASE_API_VERSION 16 // set options to API_OPTIONS to ensure compatibility #ifndef API_OPTIONS diff --git a/src/framework/cursor.cc b/src/framework/cursor.cc index 9f3e3591c..06d47d3d9 100644 --- a/src/framework/cursor.cc +++ b/src/framework/cursor.cc @@ -43,9 +43,12 @@ Cursor::Cursor(const Cursor& rhs) { name = rhs.name; buf = rhs.buf; - sz = rhs.sz; - pos = rhs.pos; file_pos = rhs.file_pos; + buf_size = rhs.buf_size; + current_pos = rhs.current_pos; + extensible = rhs.extensible; + buf_id = rhs.buf_id; + is_accumulated = rhs.is_accumulated; if (rhs.data) { @@ -107,6 +110,130 @@ void Cursor::reset(Packet* p) return; } } - set("pkt_data", p->data, p->get_detect_limit()); + + set("pkt_data", p->packet_flags & (PKT_FROM_SERVER | PKT_FROM_CLIENT), + p->data, p->get_detect_limit(), true); } +//------------------------------------------------------------------------- +// UNIT TESTS +//------------------------------------------------------------------------- +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +TEST_CASE("Boundaries", "[cursor]") +{ + const uint8_t buf_1[] = "the first"; + const uint8_t buf_2[] = "the second"; + const uint8_t buf_3[] = "the third"; + + Cursor cursor; + + SECTION("Stateless buffer") + { + const int offset = 11; + bool r1, r2; + + cursor.set("1", buf_1, sizeof(buf_1), false); + r1 = cursor.set_pos(offset); + r2 = cursor.awaiting_data(); + + CHECK(!r1); + CHECK(!r2); + } + + SECTION("Ends within 1st PDU") + { + const int offset = 8; + bool r1, r2; + + cursor.set("1", buf_1, sizeof(buf_1), true); + r1 = cursor.set_pos(offset); + r2 = cursor.awaiting_data(); + + CHECK(r1); + CHECK(!r2); + } + + SECTION("At the very end of 1st PDU") + { + const int offset = sizeof(buf_1); + bool r1, r2; + + cursor.set("1", buf_1, sizeof(buf_1), true); + r1 = cursor.set_pos(offset); + r2 = cursor.awaiting_data(); + + CHECK(r1); + CHECK(r2); + + auto rem = cursor.get_next_pos(); + CHECK(rem == 0); + + cursor.set("2", buf_2, sizeof(buf_2), true); + r1 = cursor.set_pos(rem); + r2 = cursor.awaiting_data(); + + CHECK(r1); + CHECK(!r2); + } + + SECTION("Ends after 1st PDU") + { + const int offset = 11; + bool r1, r2; + + cursor.set("1", buf_1, sizeof(buf_1), true); + r1 = cursor.set_pos(offset); + r2 = cursor.awaiting_data(); + + CHECK(!r1); + CHECK(r2); + + auto rem = cursor.get_next_pos(); + CHECK(rem == offset - sizeof(buf_1)); + + cursor.set("2", buf_2, sizeof(buf_2), true); + r1 = cursor.set_pos(rem); + r2 = cursor.awaiting_data(); + + CHECK(r1); + CHECK(!r2); + } + + SECTION("Ends after 2nd PDU") + { + const int offset = 25; + bool r1, r2; + + cursor.set("1", buf_1, sizeof(buf_1), true); + r1 = cursor.set_pos(offset); + r2 = cursor.awaiting_data(); + + CHECK(!r1); + CHECK(r2); + + auto rem1 = cursor.get_next_pos(); + CHECK(rem1 == offset - sizeof(buf_1)); + + cursor.set("2", buf_2, sizeof(buf_2), true); + r1 = cursor.set_pos(rem1); + r2 = cursor.awaiting_data(); + + CHECK(!r1); + CHECK(r2); + + auto rem2 = cursor.get_next_pos(); + CHECK(rem2 == rem1 - sizeof(buf_2)); + + cursor.set("3", buf_3, sizeof(buf_3), true); + r1 = cursor.set_pos(rem2); + r2 = cursor.awaiting_data(); + + CHECK(r1); + CHECK(!r2); + } +} + +#endif diff --git a/src/framework/cursor.h b/src/framework/cursor.h index 4954f0bd8..1a38997ba 100644 --- a/src/framework/cursor.h +++ b/src/framework/cursor.h @@ -25,6 +25,7 @@ // Cursor provides a formal way of using buffers when doing detection with // IpsOptions. +#include #include #include #include @@ -82,34 +83,55 @@ public: void reset(snort::Packet*); - void set(const char* s, const uint8_t* b, unsigned n) - { name = s; buf = b; sz = n; pos = delta = 0; } + void set(const char* s, const uint8_t* b, unsigned n, bool ext = false) + { + name = s; buf = b; buf_size = n; current_pos = delta = 0; + extensible = ext and n > 0; + buf_id = 0; + } - void set(const char* s, const uint8_t* b, unsigned n, unsigned pos_file) + void set(const char* s, const uint8_t* b, unsigned n, unsigned pos_file, bool ext = false) { file_pos = pos_file; - name = s; buf = b; sz = n; pos = delta = 0; + name = s; buf = b; buf_size = n; current_pos = delta = 0; + extensible = ext and n > 0; + buf_id = 0; + } + + void set(const char* s, uint64_t id, const uint8_t* b, unsigned n, bool ext = false) + { + set(s, b, n, ext); + buf_id = id; + } + + void set(const char* s, uint64_t id, const uint8_t* b, unsigned n, unsigned pos_file, bool ext = false) + { + set(s, b, n, pos_file, ext); + buf_id = id; } + uint64_t id() const + { return buf_id; } + const uint8_t* buffer() const { return buf; } unsigned size() const - { return sz; } + { return buf_size; } // the NEXT octect after last in buffer // (this pointer is out of bounds) const uint8_t* endo() const - { return buf + sz; } + { return buf + buf_size; } const uint8_t* start() const - { return buf + pos; } + { return buf + current_pos; } unsigned length() const - { return sz - pos; } + { return buf_size - current_pos; } unsigned get_pos() const - { return pos; } + { return current_pos; } unsigned get_delta() const { return delta; } @@ -118,19 +140,15 @@ public: bool add_pos(unsigned n) { - if (pos + n > sz) - return false; - pos += n; - return true; + current_pos += n; + return !(current_pos > buf_size); } - // pos and delta may go 1 byte after end + // current_pos and delta may go 1 byte after end bool set_pos(unsigned n) { - if (n > sz) - return false; - pos = n; - return true; + current_pos = n; + return !(current_pos > buf_size); } bool set_pos_file(unsigned n) @@ -139,14 +157,25 @@ public: return true; } + bool set_accumulation(bool is_accum) + { + is_accumulated = is_accum; + return true; + } + unsigned get_file_pos() const { return file_pos; } + bool is_buffer_accumulated() const + { + return is_accumulated; + } + bool set_delta(unsigned n) { - if (n > sz) + if (n > buf_size) return false; delta = n; return true; @@ -154,16 +183,31 @@ public: void set_data(CursorData* cd); + bool awaiting_data() const + { return extensible and current_pos >= buf_size; } + + bool awaiting_data(bool force_ext) const + { return force_ext and current_pos >= buf_size; } + + unsigned get_next_pos() const + { + assert(current_pos >= buf_size); + return current_pos - buf_size; + } + typedef std::vector CursorDataVec; private: - const char* name = nullptr; // rule option name ("pkt_data", "http_uri", etc.) - const uint8_t* buf = nullptr; // start of buffer - unsigned sz = 0; // size of buffer - unsigned pos = 0; // current pos + unsigned buf_size = 0; + unsigned current_pos = 0; unsigned delta = 0; // loop offset unsigned file_pos = 0; // file pos + const uint8_t* buf = nullptr; + const char* name = nullptr; // rule option name ("pkt_data", "http_uri", etc.) CursorDataVec* data = nullptr; // data stored on the cursor + bool extensible = false; // if the buffer could have more data in a continuation + uint64_t buf_id = 0; // source buffer ID + bool is_accumulated = false; }; #endif diff --git a/src/framework/inspector.h b/src/framework/inspector.h index 378993de6..bcd25be5e 100644 --- a/src/framework/inspector.h +++ b/src/framework/inspector.h @@ -57,6 +57,7 @@ struct InspectionBuffer }; const uint8_t* data; unsigned len; + bool is_accumulated = false; }; struct InspectApi; diff --git a/src/ips_options/ips_file_data.cc b/src/ips_options/ips_file_data.cc index 7e2ce3219..57ddaab06 100644 --- a/src/ips_options/ips_file_data.cc +++ b/src/ips_options/ips_file_data.cc @@ -56,12 +56,17 @@ IpsOption::EvalStatus FileDataOption::eval(Cursor& c, Packet* p) { RuleProfile profile(fileDataPerfStats); - DataPointer dp = DetectionEngine::get_file_data(p->context); + uint64_t sid; + bool drop_sse; + bool no_sse; + DataPointer dp = DetectionEngine::get_file_data(p->context, sid, drop_sse, no_sse); if ( !dp.data || !dp.len ) return NO_MATCH; - c.set(s_name, dp.data, dp.len); + + c.set(s_name, sid, dp.data, dp.len, !no_sse); c.set_pos_file(p->context->file_pos); + c.set_accumulation(drop_sse); return MATCH; } diff --git a/src/latency/rule_latency.cc b/src/latency/rule_latency.cc index 224e27f74..36c91e6ec 100644 --- a/src/latency/rule_latency.cc +++ b/src/latency/rule_latency.cc @@ -63,7 +63,7 @@ struct Event Type type; typename SnortClock::duration elapsed; - detection_option_tree_root_t* root; + const detection_option_tree_root_t& root; Packet* packet; }; @@ -71,10 +71,10 @@ template class RuleTimer : public LatencyTimer { public: - RuleTimer(typename Clock::duration d, detection_option_tree_root_t* root, Packet* p) : + RuleTimer(typename Clock::duration d, const detection_option_tree_root_t& root, Packet* p) : LatencyTimer(d), root(root), packet(p) { } - detection_option_tree_root_t* root; + const detection_option_tree_root_t& root; Packet* packet; }; @@ -104,11 +104,11 @@ static inline std::ostream& operator<<(std::ostream& os, const Event& e) } os << clock_usecs(TO_USECS(e.elapsed)) << " usec, "; - os << e.root->otn->sigInfo.gid << ":" << e.root->otn->sigInfo.sid << ":" - << e.root->otn->sigInfo.rev; + os << e.root.otn->sigInfo.gid << ":" << e.root.otn->sigInfo.sid << ":" + << e.root.otn->sigInfo.rev; - if ( e.root->num_children > 1 ) - os << " (of " << e.root->num_children << ")"; + if ( e.root.num_children > 1 ) + os << " (of " << e.root.num_children << ")"; if ( e.packet->has_ip() or e.packet->is_data() ) { @@ -142,7 +142,7 @@ struct DefaultRuleInterface // return true if rule was *reenabled* template - static bool reenable(detection_option_tree_root_t& root, Duration max_suspend_time, + static bool reenable(const detection_option_tree_root_t& root, Duration max_suspend_time, Time cur_time) { auto& state = root.latency_state[get_instance_id()]; @@ -156,7 +156,7 @@ struct DefaultRuleInterface } template - static bool timeout_and_suspend(detection_option_tree_root_t& root, unsigned threshold, + static bool timeout_and_suspend(const detection_option_tree_root_t& root, unsigned threshold, Time time, bool do_suspend) { auto& state = root.latency_state[get_instance_id()]; @@ -202,7 +202,7 @@ class Impl public: Impl(const ConfigWrapper&, EventHandler&); - bool push(detection_option_tree_root_t*, Packet*); + bool push(const detection_option_tree_root_t&, Packet*); bool pop(); bool suspended() const; @@ -218,16 +218,16 @@ inline Impl::Impl(const ConfigWrapper& cfg, EventHandler& eh) : { } template -inline bool Impl::push(detection_option_tree_root_t* root, Packet* p) +inline bool Impl::push(const detection_option_tree_root_t& root, Packet* p) { - assert(root and p); + assert(p); // FIXIT-L rule timer is pushed even if rule is not enabled (no visible side-effects) timers.emplace_back(config->max_time, root, p); if ( config->allow_reenable() ) { - if ( RuleTree::reenable(*root, config->max_suspend_time, Clock::now()) ) + if ( RuleTree::reenable(root, config->max_suspend_time, Clock::now()) ) { Event e { Event::EVENT_ENABLED, config->max_suspend_time, root, p }; event_handler.handle(e); @@ -246,7 +246,7 @@ inline bool Impl::pop() bool timed_out = false; - if ( !RuleTree::is_suspended(*timer.root) ) + if ( !RuleTree::is_suspended(timer.root) ) { timed_out = timer.timed_out(); #ifdef REG_TEST @@ -254,7 +254,7 @@ inline bool Impl::pop() #endif if ( timed_out ) { - auto suspended = RuleTree::timeout_and_suspend(*timer.root, config->suspend_threshold, + auto suspended = RuleTree::timeout_and_suspend(timer.root, config->suspend_threshold, Clock::now(), config->suspend); Event e @@ -278,7 +278,7 @@ inline bool Impl::suspended() const return false; assert(!timers.empty()); - return RuleTree::is_suspended(*timers.back().root); + return RuleTree::is_suspended(timers.back().root); } // ----------------------------------------------------------------------------- @@ -335,7 +335,7 @@ static inline Impl<>& get_impl() // rule latency interface // ----------------------------------------------------------------------------- -void RuleLatency::push(detection_option_tree_root_t* root, Packet* p) +void RuleLatency::push(const detection_option_tree_root_t& root, Packet* p) { if ( rule_latency::config->enabled() ) { @@ -437,11 +437,11 @@ struct RuleInterfaceSpy { is_suspended_called = true; return is_suspended_result; } template - static bool reenable(detection_option_tree_root_t&, Duration, Time) + static bool reenable(const detection_option_tree_root_t&, Duration, Time) { reenable_called = true; return reenable_result; } template - static bool timeout_and_suspend(detection_option_tree_root_t&, unsigned, Time, bool) + static bool timeout_and_suspend(const detection_option_tree_root_t&, unsigned, Time, bool) { timeout_and_suspend_called = true; return timeout_and_suspend_result; } }; @@ -477,7 +477,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) SECTION( "push rule" ) { - CHECK_FALSE( impl.push(&root, &pkt) ); + CHECK_FALSE( impl.push(root, &pkt) ); CHECK( event_handler.count == 0 ); CHECK( RuleInterfaceSpy::reenable_called ); } @@ -486,7 +486,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { RuleInterfaceSpy::reenable_result = true; - CHECK( impl.push(&root, &pkt) ); + CHECK( impl.push(root, &pkt) ); CHECK( event_handler.count == 1 ); CHECK( RuleInterfaceSpy::reenable_called ); } @@ -498,7 +498,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) SECTION( "push rule" ) { - CHECK_FALSE( impl.push(&root, &pkt) ); + CHECK_FALSE( impl.push(root, &pkt) ); CHECK( event_handler.count == 0 ); CHECK_FALSE( RuleInterfaceSpy::reenable_called ); } @@ -509,7 +509,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { RuleInterfaceSpy::is_suspended_result = true; - impl.push(&root, &pkt); + impl.push(root, &pkt); SECTION( "suspending of rules disabled" ) { @@ -532,7 +532,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { config.config.max_time = 1_ticks; - impl.push(&root, &pkt); + impl.push(root, &pkt); SECTION( "rule timeout" ) { diff --git a/src/latency/rule_latency.h b/src/latency/rule_latency.h index 464b0e1b0..0aad0b0ef 100644 --- a/src/latency/rule_latency.h +++ b/src/latency/rule_latency.h @@ -30,7 +30,7 @@ struct Packet; class RuleLatency { public: - static void push(detection_option_tree_root_t*, snort::Packet*); + static void push(const detection_option_tree_root_t&, snort::Packet*); static void pop(); static bool suspended(); @@ -39,7 +39,7 @@ public: class Context { public: - Context(detection_option_tree_root_t* root, snort::Packet* p) + Context(const detection_option_tree_root_t& root, snort::Packet* p) { RuleLatency::push(root, p); } ~Context() diff --git a/src/main/test/distill_verdict_stubs.h b/src/main/test/distill_verdict_stubs.h index 9870ff8ba..91a742380 100644 --- a/src/main/test/distill_verdict_stubs.h +++ b/src/main/test/distill_verdict_stubs.h @@ -168,6 +168,7 @@ void DetectionEngine::idle() { } void DetectionEngine::reset() { } void DetectionEngine::wait_for_context() { } void DetectionEngine::set_file_data(const DataPointer&) { } +void DetectionEngine::set_file_data(const DataPointer&, uint64_t, bool, bool) { } void DetectionEngine::clear_replacement() { } void DetectionEngine::disable_all(Packet*) { } unsigned get_instance_id() { return 0; } diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc index 04f9c99b4..141b671fc 100644 --- a/src/mime/file_mime_process.cc +++ b/src/mime/file_mime_process.cc @@ -582,7 +582,11 @@ const uint8_t* MimeSession::process_mime_data_paf( if ( result != DECODE_SUCCESS ) decompress_alert(); - set_file_data(decomp_buffer, decomp_buf_size); + if (session_base_file_id) + set_file_data(decomp_buffer, decomp_buf_size, get_multiprocessing_file_id()); + else + set_file_data(decomp_buffer, decomp_buf_size, file_counter); + attachment.data = decomp_buffer; attachment.length = decomp_buf_size; attachment.finished = isFileEnd(position); @@ -885,14 +889,15 @@ void MimeSession::mime_file_process(Packet* p, const uint8_t* data, int data_siz { Flow* flow = p->flow; FileFlows* file_flows = FileFlows::get_file_flows(flow); - if(!file_flows) + + if (!file_flows) return; if (continue_inspecting_file) { if (session_base_file_id) { - const FileDirection dir = upload? FILE_UPLOAD : FILE_DOWNLOAD; + const FileDirection dir = upload ? FILE_UPLOAD : FILE_DOWNLOAD; continue_inspecting_file = file_flows->file_process(p, get_file_cache_file_id(), data, data_size, file_offset, dir, get_multiprocessing_file_id(), position); } diff --git a/src/service_inspectors/dce_rpc/dce_smb2_file.cc b/src/service_inspectors/dce_rpc/dce_smb2_file.cc index fc1f84c72..9727a24d4 100644 --- a/src/service_inspectors/dce_rpc/dce_smb2_file.cc +++ b/src/service_inspectors/dce_rpc/dce_smb2_file.cc @@ -235,7 +235,7 @@ bool Dce2Smb2FileTracker::process_data(const uint32_t current_flow_key, const ui if (detection_size) { set_file_data(file_data, (detection_size > UINT16_MAX) ? - UINT16_MAX : (uint16_t)detection_size); + UINT16_MAX : (uint16_t)detection_size, file_id); file_detect(); } diff --git a/src/service_inspectors/dce_rpc/dce_smb_utils.cc b/src/service_inspectors/dce_rpc/dce_smb_utils.cc index 6d84ed20e..81c36bc6e 100644 --- a/src/service_inspectors/dce_rpc/dce_smb_utils.cc +++ b/src/service_inspectors/dce_rpc/dce_smb_utils.cc @@ -1720,7 +1720,8 @@ void DCE2_SmbProcessFileData(DCE2_SmbSsnData* ssd, ((ftracker->ff_file_offset == ftracker->ff_bytes_processed) && ((file_data_depth == 0) || (ftracker->ff_bytes_processed < (uint64_t)file_data_depth)))) { - set_file_data(data_ptr, (data_len > UINT16_MAX) ? UINT16_MAX : (uint16_t)data_len); + set_file_data(data_ptr, (data_len > UINT16_MAX) ? UINT16_MAX : (uint16_t)data_len, + ftracker->file_key.file_id); DCE2_FileDetect(); set_file_data(nullptr, 0); } diff --git a/src/service_inspectors/ftp_telnet/ftp_data.cc b/src/service_inspectors/ftp_telnet/ftp_data.cc index b87c257e3..68d02254e 100644 --- a/src/service_inspectors/ftp_telnet/ftp_data.cc +++ b/src/service_inspectors/ftp_telnet/ftp_data.cc @@ -62,7 +62,7 @@ static void FTPDataProcess( { int status; - set_file_data(p->data, p->dsize); + set_file_data(p->data, p->dsize, data_ssn->path_hash); if (data_ssn->packet_flags & FTPDATA_FLG_REST) { diff --git a/src/service_inspectors/http_inspect/http_field.cc b/src/service_inspectors/http_inspect/http_field.cc index c52735a85..36b384896 100644 --- a/src/service_inspectors/http_inspect/http_field.cc +++ b/src/service_inspectors/http_inspect/http_field.cc @@ -74,6 +74,7 @@ void Field::reset() strt = nullptr; len = STAT_NOT_COMPUTE; own_the_buffer = false; + was_accumulated = false; } #ifdef REG_TEST diff --git a/src/service_inspectors/http_inspect/http_field.h b/src/service_inspectors/http_inspect/http_field.h index 5abae6fa8..afc1dbce6 100644 --- a/src/service_inspectors/http_inspect/http_field.h +++ b/src/service_inspectors/http_inspect/http_field.h @@ -51,6 +51,8 @@ public: void set(HttpCommon::StatusCode stat_code); void set(int32_t length) { set(static_cast(length)); } void reset(); + void set_accumulation(bool is_accum) { was_accumulated = is_accum; } + bool is_accumulated() const { return was_accumulated; } #ifdef REG_TEST void print(FILE* output, const char* name) const; @@ -60,6 +62,8 @@ private: const uint8_t* strt = nullptr; int32_t len = HttpCommon::STAT_NOT_COMPUTE; bool own_the_buffer = false; + // FIXIT-M: find better place for the attribute, replace it with actual number of bytes processed + bool was_accumulated = false; }; struct MimeBufs diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index 8d248e2da..677ed8c6e 100755 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -261,6 +261,8 @@ bool HttpInspect::get_buf(unsigned id, Packet* p, InspectionBuffer& b) b.data = http_buffer.start(); b.len = http_buffer.length(); + b.is_accumulated = http_buffer.is_accumulated(); + return true; } diff --git a/src/service_inspectors/http_inspect/http_js_norm.cc b/src/service_inspectors/http_inspect/http_js_norm.cc index 10d99a470..702669558 100644 --- a/src/service_inspectors/http_inspect/http_js_norm.cc +++ b/src/service_inspectors/http_inspect/http_js_norm.cc @@ -246,6 +246,8 @@ void HttpJsNorm::do_external(const Field& input, Field& output, *infractions += INF_JS_CLOSING_TAG; events->create_event(EVENT_JS_CLOSING_TAG); } + if (js_ctx.is_buffer_adjusted()) + output.set_accumulation(true); if (ssn->js_built_in_event) break; @@ -391,6 +393,8 @@ void HttpJsNorm::do_inline(const Field& input, Field& output, *infractions += INF_JS_OPENING_TAG; events->create_event(EVENT_JS_OPENING_TAG); } + if (js_ctx.is_buffer_adjusted()) + output.set_accumulation(true); script_continue = ret == JSTokenizer::SCRIPT_CONTINUE; } diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index 29bc17852..d2c46878c 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -149,7 +149,7 @@ void HttpMsgBody::analyze() msg_text_new.length() : pub_depth_remaining; pub_depth_remaining -= publish_length; } - + if (session_data->mime_state[source_id]) { // FIXIT-M this interface does not convey any indication of end of message body. If the @@ -167,6 +167,9 @@ void HttpMsgBody::analyze() session_data->partial_mime_bufs[source_id] = nullptr; last_attachment_complete = session_data->partial_mime_last_complete[source_id]; session_data->partial_mime_last_complete[source_id] = true; + + if (!mime_bufs->empty()) + mime_bufs->front().file.set_accumulation(true); } else mime_bufs = new std::list; @@ -210,6 +213,8 @@ void HttpMsgBody::analyze() } else mime_bufs->emplace_back(attach_length, attach_buf, true, STAT_NOT_PRESENT, nullptr, false); + + mime_bufs->back().file.set_accumulation(!last_attachment_complete); } last_attachment_complete = latest_attachment.finished; } @@ -237,6 +242,8 @@ void HttpMsgBody::analyze() if (partial_detect_length > 0) { + detect_data.set_accumulation(true); + norm_js_data.set_accumulation(true); const int32_t total_length = partial_detect_length + decompressed_file_body.length(); assert(total_length <= @@ -288,7 +295,9 @@ void HttpMsgBody::analyze() partial_js_detect_length = js_norm_body.length(); } - set_file_data(const_cast(detect_data.start()), (unsigned)detect_data.length()); + const uint64_t file_index = get_header(source_id)->get_multi_file_processing_id(); + set_file_data(const_cast(detect_data.start()), + (unsigned)detect_data.length(), file_index, detect_data.is_accumulated()); } } body_octets += msg_text.length(); @@ -348,7 +357,7 @@ void HttpMsgBody::get_ole_data() session_data->fd_state[source_id]->ole_data_reset(); } } - + void HttpMsgBody::do_file_decompression(const Field& input, Field& output) { if (session_data->fd_state[source_id] == nullptr) @@ -569,10 +578,13 @@ bool HttpMsgBody::run_detection(snort::Packet* p) if ((mime_bufs != nullptr) && !mime_bufs->empty()) { auto mb = mime_bufs->cbegin(); - for (uint32_t count = 1; (count <= params->max_mime_attach) && (mb != mime_bufs->cend()); - count++, mb++) + for (uint32_t count = 0; (count < params->max_mime_attach) && (mb != mime_bufs->cend()); + ++count, ++mb) { - set_file_data(mb->file.start(), mb->file.length()); + const uint64_t idx = get_header(source_id)->get_multi_file_processing_id(); + set_file_data(mb->file.start(), mb->file.length(), idx, + count or mb->file.is_accumulated(), + std::next(mb) != mime_bufs->end() or last_attachment_complete); if (mb->vba.length() > 0) ole_data.set(mb->vba.length(), mb->vba.start()); DetectionEngine::detect(p); @@ -639,7 +651,7 @@ void HttpMsgBody::get_file_info(FileDirection dir, const uint8_t*& filename_buff if (filename_length > 0) return; - const Field& path = http_uri->get_norm_path(); + const Field& path = http_uri->get_norm_path(); if (path.length() > 0) { int last_slash_index = path.length() - 1; @@ -747,4 +759,3 @@ void HttpMsgBody::print_body_section(FILE* output, const char* body_type_str) HttpMsgSection::print_section_wrapup(output); } #endif - diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index b3a68f2ce..15bda008e 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -556,6 +556,9 @@ void HttpMsgHeader::setup_file_processing() if (session_data->mime_state[source_id]) return; + // Generate the unique file id for multi file processing and set ID for file_data buffer + set_multi_file_processing_id(get_transaction_id(), session_data->get_hx_stream_id()); + session_data->file_octets[source_id] = 0; const int64_t max_file_depth = FileService::get_max_file_depth(); if (max_file_depth <= 0) @@ -564,9 +567,6 @@ void HttpMsgHeader::setup_file_processing() return; } - // Generate the unique file id for multi file processing - set_multi_file_processing_id(get_transaction_id(), session_data->get_hx_stream_id()); - session_data->file_depth_remaining[source_id] = max_file_depth; FileFlows* file_flows = FileFlows::get_file_flows(flow); if (!file_flows) diff --git a/src/service_inspectors/http_inspect/ips_http_buffer.cc b/src/service_inspectors/http_inspect/ips_http_buffer.cc index a2be675e5..989f4ee34 100644 --- a/src/service_inspectors/http_inspect/ips_http_buffer.cc +++ b/src/service_inspectors/http_inspect/ips_http_buffer.cc @@ -31,8 +31,10 @@ #include "protocols/packet.h" #include "http_common.h" +#include "http_context_data.h" #include "http_enum.h" #include "http_inspect.h" +#include "http_msg_section.h" using namespace snort; using namespace HttpCommon; @@ -163,7 +165,19 @@ IpsOption::EvalStatus HttpBufferIpsOption::eval(Cursor& c, Packet* p) if (http_buffer.length() <= 0) return NO_MATCH; - c.set(key, http_buffer.start(), http_buffer.length()); + if (idx != BUFFER_PSI_JS_DATA) + { + c.set(key, http_buffer.start(), http_buffer.length()); + c.set_accumulation(http_buffer.is_accumulated()); + } + else + { + HttpMsgSection* section = HttpContextData::get_snapshot(p); + uint64_t tid = section ? section->get_transaction_id() : 0; + c.set(key, tid, http_buffer.start(), http_buffer.length(), true); + c.set_accumulation(http_buffer.is_accumulated()); + } + return MATCH; } diff --git a/src/service_inspectors/imap/imap.cc b/src/service_inspectors/imap/imap.cc index afb6a8055..0331258ad 100644 --- a/src/service_inspectors/imap/imap.cc +++ b/src/service_inspectors/imap/imap.cc @@ -177,7 +177,7 @@ static IMAPData* SetNewIMAPData(IMAP_PROTO_CONF* config, Packet* p) imap_ssn = &fd->session; imapstats.sessions++; - imap_ssn->mime_ssn= new ImapMime(p, &(config->decode_conf),&(config->log_config)); + imap_ssn->mime_ssn= new ImapMime(p, &(config->decode_conf), &(config->log_config)); imap_ssn->mime_ssn->set_mime_stats(&(imapstats.mime_stats)); if (p->packet_flags & SSNFLAG_MIDSTREAM) diff --git a/src/service_inspectors/smtp/smtp.cc b/src/service_inspectors/smtp/smtp.cc index 92e77602c..c118cf8ff 100644 --- a/src/service_inspectors/smtp/smtp.cc +++ b/src/service_inspectors/smtp/smtp.cc @@ -233,6 +233,7 @@ static SMTPData* SetNewSMTPData(SmtpProtoConf* config, Packet* p) p->flow->set_flow_data(fd); smtp_ssn = &fd->session; + smtpstats.sessions++; smtp_ssn->mime_ssn = new SmtpMime(p, &(config->decode_conf), &(config->log_config)); smtp_ssn->mime_ssn->config = config; smtp_ssn->mime_ssn->set_mime_stats(&(smtpstats.mime_stats)); diff --git a/src/stream/file/file_session.cc b/src/stream/file/file_session.cc index a65f59c6d..a3651f2fd 100644 --- a/src/stream/file/file_session.cc +++ b/src/stream/file/file_session.cc @@ -86,7 +86,7 @@ int FileSession::process(Packet* p) if (file_name) file_flows->set_file_name((const uint8_t*)file_name, strlen(file_name)); } - set_file_data(p->data, p->dsize); + set_file_data(p->data, p->dsize, c->upload); return 0; } diff --git a/src/utils/grouped_list.h b/src/utils/grouped_list.h new file mode 100644 index 000000000..8305595c6 --- /dev/null +++ b/src/utils/grouped_list.h @@ -0,0 +1,193 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// grouped_list.h author Cisco + +#ifndef GROUPED_LIST_H +#define GROUPED_LIST_H + +// The class below represents a group (double-linked list) of elements, +// where each element belongs to a sub-group. +// Sub-groups never intersect, +// but a sub-group of elements can be deleted altogether from the main group. + +#include + +namespace snort +{ + +template +class GroupedList +{ +public: + GroupedList(); + GroupedList(GroupedList& cont, GroupedList*& group, const T& value); + GroupedList(GroupedList& cont, GroupedList*& group, T&& value); + template + GroupedList(GroupedList& cont, GroupedList*& group, Args&&... args); + ~GroupedList(); + + inline T& operator *(); + inline GroupedList* get_next() const; + inline void leave_group(); + inline static unsigned erase_group(GroupedList*& group); + +private: + inline void init(); + inline void leave(); + inline void erase_all(); + + GroupedList* prev; // previous element in a container (list 1) + GroupedList* next; // next element in a container (list 1) + GroupedList* mate; // next element in a group (list 2) + GroupedList** grp; // pointer to this element, a group holder (list 2) + T value; // value of element + const bool holder; // if the element is a container holder +}; + +template +GroupedList::GroupedList() + : prev(this), next(this), mate(nullptr), grp(nullptr), holder(true) +{ } + +template +GroupedList::GroupedList(GroupedList& cont, GroupedList*& group, const T& v) + : prev(cont.prev), next(&cont), mate(group), grp(&group), value(v), holder(false) +{ + init(); + group = this; +} + +template +GroupedList::GroupedList(GroupedList& cont, GroupedList*& group, T&& v) + : prev(cont.prev), next(&cont), mate(group), grp(&group),value(v), holder(false) +{ + init(); + group = this; +} + +template template +GroupedList::GroupedList(GroupedList& cont, GroupedList*& group, Args&&... args) + : prev(cont.prev), next(&cont), mate(group), grp(&group),value{std::forward(args)...}, holder(false) +{ + init(); + group = this; +} + +template +GroupedList::~GroupedList() +{ + if (holder) + erase_all(); + else + leave(); +} + +template +T& GroupedList::operator *() +{ + return value; +} + +template +GroupedList* GroupedList::get_next() const +{ + return next; +} + +template +inline void GroupedList::leave_group() +{ + if (grp) + *grp = *grp == this ? mate : nullptr; + + if (mate) + mate->grp = grp; + + grp = nullptr; + mate = nullptr; + + assert(!holder); +} + +template +inline void GroupedList::init() +{ + assert(prev); + assert(next); + assert(next->holder); + assert(!mate or !mate->holder); + + prev->next = this; + next->prev = this; + + if (mate) + mate->grp = &mate; +} + +template +void GroupedList::leave() +{ + assert(!prev or prev->next == this); + assert(!next or next->prev == this); + + if (prev) + prev->next = next; + + if (next) + next->prev = prev; +} + +template +void GroupedList::erase_all() +{ + auto it = next; + + while (it != this) + { + auto el = it; + it = it->next; + + assert(!el->holder); + delete el; + } +} + +template +unsigned GroupedList::erase_group(GroupedList*& group) +{ + unsigned cnt = 0; + auto it = group; + group = nullptr; + + while (it) + { + auto el = it; + it = it->mate; + ++cnt; + + assert(!el->holder); + delete el; + } + + return cnt; +} + +} + +#endif diff --git a/src/utils/js_normalizer.h b/src/utils/js_normalizer.h index 5243faaa2..5de32aee8 100644 --- a/src/utils/js_normalizer.h +++ b/src/utils/js_normalizer.h @@ -71,6 +71,9 @@ public: bool is_closing_tag_seen() const { return tokenizer.is_closing_tag_seen(); } + bool is_buffer_adjusted() const + { return tokenizer.is_buffer_adjusted(); } + #if defined(CATCH_TEST_BUILD) || defined(BENCHMARK_TEST) const char* get_tmp_buf() const { return tmp_buf; } diff --git a/src/utils/js_tokenizer.h b/src/utils/js_tokenizer.h index bd46004d8..a13307409 100644 --- a/src/utils/js_tokenizer.h +++ b/src/utils/js_tokenizer.h @@ -180,6 +180,7 @@ public: bool is_mixed_encoding_seen() const; bool is_opening_tag_seen() const; bool is_closing_tag_seen() const; + bool is_buffer_adjusted() const; protected: [[noreturn]] void LexerError(const char* msg) override @@ -368,6 +369,7 @@ private: const int tmp_cap_size; bool newline_found = false; + bool adjusted_data = false; // flag for resetting the continuation in case of adjusting js_data constexpr static bool insert_semicolon[ASI_GROUP_MAX][ASI_GROUP_MAX] { {false, false, false, false, false, false, false, false, false, false, false,}, diff --git a/src/utils/js_tokenizer.l b/src/utils/js_tokenizer.l index f75731ae4..de0bcdd0d 100644 --- a/src/utils/js_tokenizer.l +++ b/src/utils/js_tokenizer.l @@ -1828,6 +1828,8 @@ bool JSTokenizer::states_process() void JSTokenizer::states_adjust() { + adjusted_data = true; + int outbuf_pos = yyout.rdbuf()->pubseekoff(0, std::ios_base::cur, std::ios_base::out); assert(outbuf_pos >= 0); @@ -2045,6 +2047,11 @@ bool JSTokenizer::is_closing_tag_seen() const return closing_tag_seen; } +bool JSTokenizer::is_buffer_adjusted() const +{ + return adjusted_data; +} + void JSTokenizer::set_block_param(bool f) { scope_cur().block_param = f; @@ -3030,6 +3037,7 @@ JSTokenizer::JSRet JSTokenizer::process(size_t& bytes_in, bool external_script) unescape_nest_seen = false; mixed_encoding_seen = false; ext_script = external_script; + adjusted_data = false; auto r = yylex(); diff --git a/src/utils/stats.cc b/src/utils/stats.cc index be430903e..0f3147f96 100644 --- a/src/utils/stats.cc +++ b/src/utils/stats.cc @@ -214,6 +214,13 @@ const PegInfo pc_names[] = { CountType::SUM, "pcre_match_limit", "total number of times pcre hit the match limit" }, { CountType::SUM, "pcre_recursion_limit", "total number of times pcre hit the recursion limit" }, { CountType::SUM, "pcre_error", "total number of times pcre returns error" }, + { CountType::SUM, "cont_creations", "total number of continuations created" }, + { CountType::SUM, "cont_recalls", "total number of continuations recalled" }, + { CountType::SUM, "cont_flows", "total number of flows using continuation" }, + { CountType::SUM, "cont_evals", "total number of condition-met continuations" }, + { CountType::SUM, "cont_matches", "total number of continuations matched" }, + { CountType::SUM, "cont_mismatches", "total number of continuations mismatched" }, + { CountType::MAX, "cont_max_num", "peak number of simultaneous continuations per flow" }, { CountType::END, nullptr, nullptr } }; diff --git a/src/utils/stats.h b/src/utils/stats.h index 8e548b062..808bfffb4 100644 --- a/src/utils/stats.h +++ b/src/utils/stats.h @@ -63,6 +63,13 @@ struct PacketCount PegCount pcre_match_limit; PegCount pcre_recursion_limit; PegCount pcre_error; + PegCount cont_creations; + PegCount cont_recalls; + PegCount cont_flows; + PegCount cont_evals; + PegCount cont_matches; + PegCount cont_mismatches; + PegCount cont_max_num; }; struct ProcessCount diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index b9be76201..12a694b3b 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -66,3 +66,7 @@ add_catch_test( streambuf_test ../streambuf.cc ) +add_catch_test( grouped_list_test + SOURCES + ../grouped_list.h +) diff --git a/src/utils/test/grouped_list_test.cc b/src/utils/test/grouped_list_test.cc new file mode 100644 index 000000000..9dd79539c --- /dev/null +++ b/src/utils/test/grouped_list_test.cc @@ -0,0 +1,595 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// grouped_list_test.cc author Cisco + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "catch/catch.hpp" + +#include +#include + +#include "utils/grouped_list.h" + +using namespace snort; +using namespace std; + +struct Type +{ + int id = 0; + const char* name = ""; + + bool operator ==(const Type& r) const + { return id == r.id and !strcmp(name, r.name); } +}; + +using Elem = GroupedList; + +template +static void check_container(const Elem& cont, Args&& ... args) +{ + const vector data{std::forward(args)...}; + Elem* it = cont.get_next(); + + for (const auto& d : data) + { + REQUIRE(it != nullptr); + CHECK((**it) == d); + it = it->get_next(); + } + + CHECK(it == &cont); +} + +TEST_CASE("Basic", "[Double list]") +{ + Elem cont; + Elem* group = nullptr; + + SECTION("no data") + { + check_container(cont); + } + + SECTION("1 element") + { + const Type data = {1, "one"}; + Elem* el = new Elem(cont, group, data); + + CHECK(el != nullptr); + + check_container(cont, data); + } + + SECTION("3 elements") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* el1 = new Elem(cont, group, data1); + Elem* el2 = new Elem(cont, group, data2); + Elem* el3 = new Elem(cont, group, data3); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + CHECK(el3 != nullptr); + + check_container(cont, data1, data2, data3); + } +} + +TEST_CASE("Groups", "[Double list]") +{ + Elem cont; + + SECTION("3 element group") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_a, data2); + CHECK(group_a == el2); + Elem* el3 = new Elem(cont, group_a, data3); + CHECK(group_a == el3); + + check_container(cont, data1, data2, data3); + + auto cnt = Elem::erase_group(group_a); + CHECK(cnt == 3); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("3 groups") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + Elem* group_b = nullptr; + Elem* group_c = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_b, data2); + CHECK(group_b == el2); + Elem* el3 = new Elem(cont, group_c, data3); + CHECK(group_c == el3); + + check_container(cont, data1, data2, data3); + + auto cnt1 = Elem::erase_group(group_a); + CHECK(cnt1 == 1); + CHECK(group_a == nullptr); + + check_container(cont, data2, data3); + + auto cnt2 = Elem::erase_group(group_b); + CHECK(cnt2 == 1); + CHECK(group_b == nullptr); + + check_container(cont, data3); + + auto cnt3 = Elem::erase_group(group_c); + CHECK(cnt3 == 1); + CHECK(group_c == nullptr); + + check_container(cont); + } + + SECTION("interleaving groups") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + Elem* group_b = nullptr; + Elem* group_c = nullptr; + + new Elem(cont, group_a, data1); + new Elem(cont, group_b, data2); + new Elem(cont, group_c, data3); + new Elem(cont, group_a, data1); + new Elem(cont, group_b, data2); + new Elem(cont, group_c, data3); + new Elem(cont, group_a, data1); + new Elem(cont, group_b, data2); + new Elem(cont, group_c, data3); + + check_container(cont, data1, data2, data3, data1, data2, data3, data1, data2, data3); + + auto cnt1 = Elem::erase_group(group_a); + CHECK(cnt1 == 3); + CHECK(group_a == nullptr); + + check_container(cont, data2, data3, data2, data3, data2, data3); + + auto cnt2 = Elem::erase_group(group_b); + CHECK(cnt2 == 3); + CHECK(group_b == nullptr); + + check_container(cont, data3, data3, data3); + + auto cnt3 = Elem::erase_group(group_c); + CHECK(cnt3 == 3); + CHECK(group_c == nullptr); + + check_container(cont); + } + + SECTION("leaving a group (middle)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_a, data2); + CHECK(group_a == el2); + Elem* el3 = new Elem(cont, group_a, data3); + CHECK(group_a == el3); + + check_container(cont, data1, data2, data3); + + el2->leave_group(); + delete el2; + CHECK(group_a == el3); + + check_container(cont, data1, data3); + + auto cnt = Elem::erase_group(group_a); + CHECK(cnt == 2); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("leaving a group (begin)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_a, data2); + CHECK(group_a == el2); + Elem* el3 = new Elem(cont, group_a, data3); + CHECK(group_a == el3); + + check_container(cont, data1, data2, data3); + + el1->leave_group(); + delete el1; + CHECK(group_a == el3); + + check_container(cont, data2, data3); + + auto cnt = Elem::erase_group(group_a); + CHECK(cnt == 2); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("leaving a group (end)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_a, data2); + CHECK(group_a == el2); + Elem* el3 = new Elem(cont, group_a, data3); + CHECK(group_a == el3); + + check_container(cont, data1, data2, data3); + + el3->leave_group(); + delete el3; + CHECK(group_a == el2); + + check_container(cont, data1, data2); + + auto cnt = Elem::erase_group(group_a); + CHECK(cnt == 2); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("leaving a group (one by one)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + CHECK(group_a == el1); + Elem* el2 = new Elem(cont, group_a, data2); + CHECK(group_a == el2); + Elem* el3 = new Elem(cont, group_a, data3); + CHECK(group_a == el3); + + check_container(cont, data1, data2, data3); + + el3->leave_group(); + delete el3; + CHECK(group_a == el2); + + el2->leave_group(); + delete el2; + CHECK(group_a == el1); + + el1->leave_group(); + delete el1; + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("leaving a group (repeated call)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + const Type data4 = {4, "four"}; + const Type data5 = {5, "five"}; + Elem* group_a = nullptr; + + Elem* el1 = new Elem(cont, group_a, data1); + Elem* el2 = new Elem(cont, group_a, data2); + Elem* el3 = new Elem(cont, group_a, data3); + Elem* el4 = new Elem(cont, group_a, data4); + Elem* el5 = new Elem(cont, group_a, data5); + + check_container(cont, data1, data2, data3, data4, data5); + + el1->leave_group(); + el1->leave_group(); + el1->leave_group(); + + el3->leave_group(); + el3->leave_group(); + el3->leave_group(); + + el5->leave_group(); + el5->leave_group(); + el5->leave_group(); + + CHECK(group_a == el4); + check_container(cont, data1, data2, data3, data4, data5); + + delete el1; + delete el2; + delete el3; + delete el4; + delete el5; + } +} + +TEST_CASE("Memory management (value by copy)", "[Double list]") +{ + Elem cont; + Elem* group_a = nullptr; + Elem* group_b = nullptr; + + SECTION("delete element") + { + const Type data = {1, "one"}; + Elem* el = new Elem(cont, group_a, data); + + CHECK(el != nullptr); + + check_container(cont, data); + + delete el; + + check_container(cont); + } + + SECTION("delete group") + { + const Type data = {1, "one"}; + Elem* el = new Elem(cont, group_a, data); + + CHECK(el != nullptr); + + check_container(cont, data); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("delete element, then delete group") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + Elem* el1 = new Elem(cont, group_a, data1); + Elem* el2 = new Elem(cont, group_b, data2); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + + delete el2; + + check_container(cont, data1); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont); + + } + + SECTION("delete group, then delete element") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + Elem* el1 = new Elem(cont, group_a, data1); + Elem* el2 = new Elem(cont, group_b, data2); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, data2); + + delete el2; + + check_container(cont); + } + + SECTION("delete elements (remain), then delete group") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* el1 = new Elem(cont, group_a, data1); + Elem* el2 = new Elem(cont, group_b, data2); + Elem* el3 = new Elem(cont, group_b, data3); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + CHECK(el3 != nullptr); + + delete el2; + + check_container(cont, data1, data3); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, data3); + + } + + SECTION("delete group, then delete elements (remain)") + { + const Type data1 = {1, "one"}; + const Type data2 = {2, "two"}; + const Type data3 = {3, "three"}; + Elem* el1 = new Elem(cont, group_a, data1); + Elem* el2 = new Elem(cont, group_b, data2); + Elem* el3 = new Elem(cont, group_b, data3); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + CHECK(el3 != nullptr); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, data2, data3); + + delete el2; + + check_container(cont, data3); + } +} + +TEST_CASE("Memory management (value in-place)", "[Double list]") +{ + Elem cont; + Elem* group_a = nullptr; + Elem* group_b = nullptr; + + SECTION("delete element") + { + Elem* el = new Elem(cont, group_a, 1, "one"); + + CHECK(el != nullptr); + + check_container(cont, Type{1, "one"}); + + delete el; + + check_container(cont); + } + + SECTION("delete group") + { + Elem* el = new Elem(cont, group_a, 1, "one"); + + CHECK(el != nullptr); + + check_container(cont, Type{1, "one"}); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont); + } + + SECTION("delete element, then delete group") + { + Elem* el1 = new Elem(cont, group_a, 1, "one"); + Elem* el2 = new Elem(cont, group_b, 2, "two"); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + + delete el2; + + check_container(cont, Type{1, "one"}); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont); + + } + + SECTION("delete group, then delete element") + { + Elem* el1 = new Elem(cont, group_a, 1, "one"); + Elem* el2 = new Elem(cont, group_b, 2, "two"); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, Type{2, "two"}); + + delete el2; + + check_container(cont); + } + + SECTION("delete elements (remain), then delete group") + { + Elem* el1 = new Elem(cont, group_a, 1, "one"); + Elem* el2 = new Elem(cont, group_b, 2, "two"); + Elem* el3 = new Elem(cont, group_b, 3, "three"); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + CHECK(el3 != nullptr); + + delete el2; + + check_container(cont, Type{1, "one"}, Type{3, "three"}); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, Type{3, "three"}); + + } + + SECTION("delete group, then delete elements (remain)") + { + Elem* el1 = new Elem(cont, group_a, 1, "one"); + Elem* el2 = new Elem(cont, group_b, 2, "two"); + Elem* el3 = new Elem(cont, group_b, 3, "three"); + + CHECK(el1 != nullptr); + CHECK(el2 != nullptr); + CHECK(el3 != nullptr); + + Elem::erase_group(group_a); + CHECK(group_a == nullptr); + + check_container(cont, Type{2, "two"}, Type{3, "three"}); + + delete el2; + + check_container(cont, Type{3, "three"}); + } +}