From: Russ Combs (rucombs) Date: Wed, 3 Feb 2016 21:09:21 +0000 (-0500) Subject: Merge pull request #236 in SNORT/snort3 from nhttp36 to master X-Git-Tag: 3.0.0-233~638 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7627f57735f339f49e55c8eb19630225c07f46c5;p=thirdparty%2Fsnort3.git Merge pull request #236 in SNORT/snort3 from nhttp36 to master Squashed commit of the following: commit 3e21cabcf4561d48811a7a0b7cff3ac0ac687c50 Author: Tom Peters Date: Wed Feb 3 12:27:25 2016 -0500 nhttp_stream_splitter.cc subdivided --- diff --git a/src/service_inspectors/nhttp_inspect/CMakeLists.txt b/src/service_inspectors/nhttp_inspect/CMakeLists.txt index 50750190c..71484325d 100644 --- a/src/service_inspectors/nhttp_inspect/CMakeLists.txt +++ b/src/service_inspectors/nhttp_inspect/CMakeLists.txt @@ -50,7 +50,8 @@ set (FILE_LIST nhttp_enum.h nhttp_field.cc nhttp_field.h - nhttp_stream_splitter.cc + nhttp_stream_splitter_reassemble.cc + nhttp_stream_splitter_scan.cc nhttp_stream_splitter.h nhttp_cutter.cc nhttp_cutter.h diff --git a/src/service_inspectors/nhttp_inspect/Makefile.am b/src/service_inspectors/nhttp_inspect/Makefile.am index c852c8f44..7f1b39b44 100644 --- a/src/service_inspectors/nhttp_inspect/Makefile.am +++ b/src/service_inspectors/nhttp_inspect/Makefile.am @@ -24,7 +24,7 @@ nhttp_module.cc nhttp_module.h \ nhttp_test_input.cc nhttp_test_input.h \ nhttp_flow_data.cc nhttp_flow_data.h \ nhttp_transaction.cc nhttp_transaction.h \ -nhttp_stream_splitter.cc nhttp_stream_splitter.h \ +nhttp_stream_splitter_reassemble.cc nhttp_stream_splitter_scan.cc nhttp_stream_splitter.h \ nhttp_cutter.cc nhttp_cutter.h \ nhttp_enum.h \ nhttp_test_manager.cc nhttp_test_manager.h \ diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_reassemble.cc similarity index 62% rename from src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc rename to src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_reassemble.cc index 27516d5a3..fc7b7f08a 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_reassemble.cc @@ -15,7 +15,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- -// nhttp_stream_splitter.cc author Tom Peters +// nhttp_stream_splitter_reassemble.cc author Tom Peters #include #include @@ -25,58 +25,11 @@ #include "nhttp_field.h" #include "nhttp_test_manager.h" #include "nhttp_test_input.h" -#include "nhttp_cutter.h" #include "nhttp_inspect.h" #include "nhttp_stream_splitter.h" using namespace NHttpEnums; -// Convenience function. All housekeeping that must be done before we can return FLUSH to stream. -void NHttpStreamSplitter::prepare_flush(NHttpFlowData* session_data, uint32_t* flush_offset, - SectionType section_type, uint32_t num_flushed, uint32_t num_excess, int32_t num_head_lines, - bool is_broken_chunk, uint32_t num_good_chunks) const -{ - session_data->section_type[source_id] = section_type; - session_data->num_excess[source_id] = num_excess; - session_data->num_head_lines[source_id] = num_head_lines; - session_data->is_broken_chunk[source_id] = is_broken_chunk; - session_data->num_good_chunks[source_id] = num_good_chunks; - -#ifdef REG_TEST - if (NHttpTestManager::use_test_input()) - { - NHttpTestManager::get_test_input_source()->flush(num_flushed); - } - else -#endif - - *flush_offset = num_flushed; -} - -NHttpCutter* NHttpStreamSplitter::get_cutter(SectionType type, - const NHttpFlowData* session_data) const -{ - switch (type) - { - case SEC_REQUEST: - return (NHttpCutter*)new NHttpRequestCutter; - case SEC_STATUS: - return (NHttpCutter*)new NHttpStatusCutter; - case SEC_HEADER: - case SEC_TRAILER: - return (NHttpCutter*)new NHttpHeaderCutter; - case SEC_BODY_CL: - return (NHttpCutter*)new NHttpBodyClCutter(session_data->data_length[source_id]); - case SEC_BODY_CHUNK: - return (NHttpCutter*)new NHttpBodyChunkCutter; - case SEC_BODY_OLD: - return (NHttpCutter*)new NHttpBodyOldCutter; - default: - assert(false); - return nullptr; - } -} - void NHttpStreamSplitter::chunk_spray(NHttpFlowData* session_data, uint8_t* buffer, const uint8_t* data, unsigned length) const { @@ -245,117 +198,6 @@ void NHttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, con offset += length; } -StreamSplitter::Status NHttpStreamSplitter::scan(Flow* flow, const uint8_t* data, uint32_t length, - uint32_t, uint32_t* flush_offset) -{ - assert(length <= MAX_OCTETS); - - // This is the session state information we share with NHttpInspect and store with stream. A - // session is defined by a TCP connection. Since scan() is the first to see a new TCP - // connection the new flow data object is created here. - NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data( - NHttpFlowData::nhttp_flow_id); - if (session_data == nullptr) - { - flow->set_application_data(session_data = new NHttpFlowData); - } - assert(session_data != nullptr); - - const SectionType type = session_data->type_expected[source_id]; - - if (type == SEC_ABORT) - return StreamSplitter::ABORT; - -#ifdef REG_TEST - if (NHttpTestManager::use_test_input()) - { - // This block substitutes a completely new data buffer supplied by the test tool in place - // of the "real" data. It also rewrites the buffer length. - *flush_offset = length; - uint8_t* test_data = nullptr; - NHttpTestManager::get_test_input_source()->scan(test_data, length, source_id, - session_data->seq_num); - if (length == 0) - return StreamSplitter::FLUSH; - data = test_data; - } - else if (NHttpTestManager::use_test_output()) - { - printf("Scan from flow data %" PRIu64 " direction %d length %u\n", session_data->seq_num, - source_id, length); - fflush(stdout); - } -#endif - - assert(!session_data->tcp_close[source_id]); - - NHttpCutter*& cutter = session_data->cutter[source_id]; - if (cutter == nullptr) - { - cutter = get_cutter(type, session_data); - assert(cutter != nullptr); - } - const uint32_t max_length = MAX_OCTETS - cutter->get_octets_seen(); - const ScanResult cut_result = cutter->cut(data, (length <= max_length) ? length : - max_length, session_data->infractions[source_id], session_data->events[source_id], - session_data->section_size_target[source_id], session_data->section_size_max[source_id]); - switch (cut_result) - { - case SCAN_NOTFOUND: - if (cutter->get_octets_seen() == MAX_OCTETS) - { - session_data->infractions[source_id] += INF_ENDLESS_HEADER; - // FIXIT-H suspicion that this alert is never generated. - session_data->events[source_id].create_event(EVENT_LOSS_OF_SYNC); - // FIXIT-H need to process this data not just discard it. - session_data->type_expected[source_id] = SEC_ABORT; - delete cutter; - cutter = nullptr; - return StreamSplitter::ABORT; - } - // Incomplete headers wait patiently for more data -#ifdef REG_TEST - if (NHttpTestManager::use_test_input()) - return StreamSplitter::FLUSH; - else -#endif - return StreamSplitter::SEARCH; - case SCAN_ABORT: - case SCAN_END: // FIXIT-H need StreamSplitter::END - session_data->type_expected[source_id] = SEC_ABORT; - delete cutter; - cutter = nullptr; - return StreamSplitter::ABORT; - case SCAN_DISCARD: - case SCAN_DISCARD_PIECE: - prepare_flush(session_data, flush_offset, SEC_DISCARD, cutter->get_num_flush(), 0, 0, - false, 0); - if (cut_result == SCAN_DISCARD) - { - delete cutter; - cutter = nullptr; - } - return StreamSplitter::FLUSH; - case SCAN_FOUND: - case SCAN_FOUND_PIECE: - { - const uint32_t flush_octets = cutter->get_num_flush(); - prepare_flush(session_data, flush_offset, type, flush_octets, cutter->get_num_excess(), - cutter->get_num_head_lines(), cutter->get_is_broken_chunk(), - cutter->get_num_good_chunks()); - if (cut_result == SCAN_FOUND) - { - delete cutter; - cutter = nullptr; - } - return StreamSplitter::FLUSH; - } - default: - assert(false); - return StreamSplitter::ABORT; - } -} - const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned total, unsigned, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) { @@ -520,73 +362,3 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned total, return nullptr; } -bool NHttpStreamSplitter::finish(Flow* flow) -{ - NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data( - NHttpFlowData::nhttp_flow_id); - - assert(session_data != nullptr); - -#ifdef REG_TEST - if (NHttpTestManager::use_test_output() && !NHttpTestManager::use_test_input()) - { - printf("Finish from flow data %" PRIu64 " direction %d\n", session_data->seq_num, - source_id); - fflush(stdout); - } -#endif - - if (session_data->type_expected[source_id] == SEC_ABORT) - { - return false; - } - - session_data->tcp_close[source_id] = true; - - // If there is leftover data for which we returned PAF_SEARCH and never flushed, we need to set - // up to process because it is about to go to reassemble(). But we don't support partial start - // lines. - if ((session_data->section_type[source_id] == SEC__NOT_COMPUTE) && - (session_data->cutter[source_id] != nullptr) && - (session_data->cutter[source_id]->get_octets_seen() > 0)) - { - if ((session_data->type_expected[source_id] == SEC_REQUEST) || - (session_data->type_expected[source_id] == SEC_STATUS)) - { - session_data->infractions[source_id] += INF_PARTIAL_START; - session_data->events[source_id].create_event(EVENT_LOSS_OF_SYNC); - return false; - } - - uint32_t not_used; - prepare_flush(session_data, ¬_used, session_data->type_expected[source_id], 0, 0, - session_data->cutter[source_id]->get_num_head_lines() + 1, - session_data->cutter[source_id]->get_is_broken_chunk(), - session_data->cutter[source_id]->get_num_good_chunks()); - return true; - } - - // If there is no more data to process we need to wrap up file processing right now - if ((session_data->section_type[source_id] == SEC__NOT_COMPUTE) && - (session_data->file_depth_remaining[source_id] > 0) && - (session_data->cutter[source_id] != nullptr) && - (session_data->cutter[source_id]->get_octets_seen() == 0)) - { - if (source_id == SRC_SERVER) - { - FileFlows* file_flows = FileFlows::get_file_flows(flow); - file_flows->file_process(nullptr, 0, SNORT_FILE_END, false); - } - else if (session_data->mime_state != nullptr) - { - session_data->mime_state->process_mime_data(flow, nullptr, 0, true, - SNORT_FILE_END); - delete session_data->mime_state; - session_data->mime_state = nullptr; - } - return false; - } - - return true; -} - diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_scan.cc b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_scan.cc new file mode 100644 index 000000000..ea2d195dc --- /dev/null +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter_scan.cc @@ -0,0 +1,258 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// nhttp_stream_splitter_scan.cc author Tom Peters + +#include +#include + +#include "file_api/file_flows.h" +#include "nhttp_enum.h" +#include "nhttp_field.h" +#include "nhttp_test_manager.h" +#include "nhttp_test_input.h" +#include "nhttp_cutter.h" +#include "nhttp_stream_splitter.h" + +using namespace NHttpEnums; + +// Convenience function. All housekeeping that must be done before we can return FLUSH to stream. +void NHttpStreamSplitter::prepare_flush(NHttpFlowData* session_data, uint32_t* flush_offset, + SectionType section_type, uint32_t num_flushed, uint32_t num_excess, int32_t num_head_lines, + bool is_broken_chunk, uint32_t num_good_chunks) const +{ + session_data->section_type[source_id] = section_type; + session_data->num_excess[source_id] = num_excess; + session_data->num_head_lines[source_id] = num_head_lines; + session_data->is_broken_chunk[source_id] = is_broken_chunk; + session_data->num_good_chunks[source_id] = num_good_chunks; + +#ifdef REG_TEST + if (NHttpTestManager::use_test_input()) + { + NHttpTestManager::get_test_input_source()->flush(num_flushed); + } + else +#endif + + *flush_offset = num_flushed; +} + +NHttpCutter* NHttpStreamSplitter::get_cutter(SectionType type, + const NHttpFlowData* session_data) const +{ + switch (type) + { + case SEC_REQUEST: + return (NHttpCutter*)new NHttpRequestCutter; + case SEC_STATUS: + return (NHttpCutter*)new NHttpStatusCutter; + case SEC_HEADER: + case SEC_TRAILER: + return (NHttpCutter*)new NHttpHeaderCutter; + case SEC_BODY_CL: + return (NHttpCutter*)new NHttpBodyClCutter(session_data->data_length[source_id]); + case SEC_BODY_CHUNK: + return (NHttpCutter*)new NHttpBodyChunkCutter; + case SEC_BODY_OLD: + return (NHttpCutter*)new NHttpBodyOldCutter; + default: + assert(false); + return nullptr; + } +} + +StreamSplitter::Status NHttpStreamSplitter::scan(Flow* flow, const uint8_t* data, uint32_t length, + uint32_t, uint32_t* flush_offset) +{ + assert(length <= MAX_OCTETS); + + // This is the session state information we share with NHttpInspect and store with stream. A + // session is defined by a TCP connection. Since scan() is the first to see a new TCP + // connection the new flow data object is created here. + NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data( + NHttpFlowData::nhttp_flow_id); + if (session_data == nullptr) + { + flow->set_application_data(session_data = new NHttpFlowData); + } + assert(session_data != nullptr); + + const SectionType type = session_data->type_expected[source_id]; + if (type == SEC_ABORT) + return StreamSplitter::ABORT; + +#ifdef REG_TEST + if (NHttpTestManager::use_test_input()) + { + // This block substitutes a completely new data buffer supplied by the test tool in place + // of the "real" data. It also rewrites the buffer length. + *flush_offset = length; + uint8_t* test_data = nullptr; + NHttpTestManager::get_test_input_source()->scan(test_data, length, source_id, + session_data->seq_num); + if (length == 0) + return StreamSplitter::FLUSH; + data = test_data; + } + else if (NHttpTestManager::use_test_output()) + { + printf("Scan from flow data %" PRIu64 " direction %d length %u\n", session_data->seq_num, + source_id, length); + fflush(stdout); + } +#endif + + assert(!session_data->tcp_close[source_id]); + + NHttpCutter*& cutter = session_data->cutter[source_id]; + if (cutter == nullptr) + { + cutter = get_cutter(type, session_data); + assert(cutter != nullptr); + } + const uint32_t max_length = MAX_OCTETS - cutter->get_octets_seen(); + const ScanResult cut_result = cutter->cut(data, (length <= max_length) ? length : + max_length, session_data->infractions[source_id], session_data->events[source_id], + session_data->section_size_target[source_id], session_data->section_size_max[source_id]); + switch (cut_result) + { + case SCAN_NOTFOUND: + if (cutter->get_octets_seen() == MAX_OCTETS) + { + session_data->infractions[source_id] += INF_ENDLESS_HEADER; + // FIXIT-H suspicion that this alert is never generated. + session_data->events[source_id].create_event(EVENT_LOSS_OF_SYNC); + // FIXIT-H need to process this data not just discard it. + session_data->type_expected[source_id] = SEC_ABORT; + delete cutter; + cutter = nullptr; + return StreamSplitter::ABORT; + } + // Incomplete headers wait patiently for more data +#ifdef REG_TEST + if (NHttpTestManager::use_test_input()) + return StreamSplitter::FLUSH; + else +#endif + return StreamSplitter::SEARCH; + case SCAN_ABORT: + case SCAN_END: // FIXIT-H need StreamSplitter::END + session_data->type_expected[source_id] = SEC_ABORT; + delete cutter; + cutter = nullptr; + return StreamSplitter::ABORT; + case SCAN_DISCARD: + case SCAN_DISCARD_PIECE: + prepare_flush(session_data, flush_offset, SEC_DISCARD, cutter->get_num_flush(), 0, 0, + false, 0); + if (cut_result == SCAN_DISCARD) + { + delete cutter; + cutter = nullptr; + } + return StreamSplitter::FLUSH; + case SCAN_FOUND: + case SCAN_FOUND_PIECE: + { + const uint32_t flush_octets = cutter->get_num_flush(); + prepare_flush(session_data, flush_offset, type, flush_octets, cutter->get_num_excess(), + cutter->get_num_head_lines(), cutter->get_is_broken_chunk(), + cutter->get_num_good_chunks()); + if (cut_result == SCAN_FOUND) + { + delete cutter; + cutter = nullptr; + } + return StreamSplitter::FLUSH; + } + default: + assert(false); + return StreamSplitter::ABORT; + } +} + +bool NHttpStreamSplitter::finish(Flow* flow) +{ + NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data( + NHttpFlowData::nhttp_flow_id); + + assert(session_data != nullptr); + +#ifdef REG_TEST + if (NHttpTestManager::use_test_output() && !NHttpTestManager::use_test_input()) + { + printf("Finish from flow data %" PRIu64 " direction %d\n", session_data->seq_num, + source_id); + fflush(stdout); + } +#endif + + if (session_data->type_expected[source_id] == SEC_ABORT) + { + return false; + } + + session_data->tcp_close[source_id] = true; + + // If there is leftover data for which we returned PAF_SEARCH and never flushed, we need to set + // up to process because it is about to go to reassemble(). But we don't support partial start + // lines. + if ((session_data->section_type[source_id] == SEC__NOT_COMPUTE) && + (session_data->cutter[source_id] != nullptr) && + (session_data->cutter[source_id]->get_octets_seen() > 0)) + { + if ((session_data->type_expected[source_id] == SEC_REQUEST) || + (session_data->type_expected[source_id] == SEC_STATUS)) + { + session_data->infractions[source_id] += INF_PARTIAL_START; + session_data->events[source_id].create_event(EVENT_LOSS_OF_SYNC); + return false; + } + + uint32_t not_used; + prepare_flush(session_data, ¬_used, session_data->type_expected[source_id], 0, 0, + session_data->cutter[source_id]->get_num_head_lines() + 1, + session_data->cutter[source_id]->get_is_broken_chunk(), + session_data->cutter[source_id]->get_num_good_chunks()); + return true; + } + + // If there is no more data to process we need to wrap up file processing right now + if ((session_data->section_type[source_id] == SEC__NOT_COMPUTE) && + (session_data->file_depth_remaining[source_id] > 0) && + (session_data->cutter[source_id] != nullptr) && + (session_data->cutter[source_id]->get_octets_seen() == 0)) + { + if (source_id == SRC_SERVER) + { + FileFlows* file_flows = FileFlows::get_file_flows(flow); + file_flows->file_process(nullptr, 0, SNORT_FILE_END, false); + } + else if (session_data->mime_state != nullptr) + { + session_data->mime_state->process_mime_data(flow, nullptr, 0, true, + SNORT_FILE_END); + delete session_data->mime_state; + session_data->mime_state = nullptr; + } + return false; + } + + return true; +} +