From: Tom Peters Date: Fri, 12 Sep 2014 19:00:15 +0000 (-0400) Subject: checkpoint after review of test cases X-Git-Tag: 3.0.0-233~1416 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=6ebfb3a4bcbd2fb6dedb593d2a9c26eb435a88bd;p=thirdparty%2Fsnort3.git checkpoint after review of test cases --- diff --git a/src/service_inspectors/nhttp_inspect/CMakeLists.txt b/src/service_inspectors/nhttp_inspect/CMakeLists.txt index 90d12897f..a702fc691 100644 --- a/src/service_inspectors/nhttp_inspect/CMakeLists.txt +++ b/src/service_inspectors/nhttp_inspect/CMakeLists.txt @@ -44,6 +44,8 @@ set (FILE_LIST nhttp_transaction.cc nhttp_transaction.h nhttp_scratch_pad.h + nhttp_test_manager.cc + nhttp_test_manager.h nhttp_enum.h nhttp_field.cc nhttp_field.h diff --git a/src/service_inspectors/nhttp_inspect/Makefile.am b/src/service_inspectors/nhttp_inspect/Makefile.am index e9669ff33..e46e68be9 100644 --- a/src/service_inspectors/nhttp_inspect/Makefile.am +++ b/src/service_inspectors/nhttp_inspect/Makefile.am @@ -25,7 +25,9 @@ nhttp_test_input.cc nhttp_test_input.h \ nhttp_flow_data.cc nhttp_flow_data.h \ nhttp_transaction.cc nhttp_transaction.h \ nhttp_stream_splitter.cc nhttp_stream_splitter.h \ -nhttp_scratch_pad.h nhttp_enum.h \ +nhttp_scratch_pad.h \ +nhttp_enum.h \ +nhttp_test_manager.cc nhttp_test_manager.h \ nhttp_field.cc nhttp_field.h if STATIC_INSPECTORS diff --git a/src/service_inspectors/nhttp_inspect/nhttp_field.cc b/src/service_inspectors/nhttp_inspect/nhttp_field.cc index f9e184764..d7654bf23 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_field.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_field.cc @@ -46,7 +46,7 @@ void Field::print(FILE *output, const char* name, bool int_vals) const { fprintf(output, "\n"); return; } - int32_t print_length = (length <= 1000) ? length : 1000; // Limit the amount of data printed + int32_t print_length = (length <= 1200) ? length : 1200; // Limit the amount of data printed for (int k=0; k < print_length; k++) { if ((start[k] >= 0x20) && (start[k] <= 0x7E)) fprintf(output, "%c", (char)start[k]); else if (start[k] == 0xD) fprintf(output, "~"); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h index 023738a5f..a1e411002 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h @@ -23,7 +23,7 @@ // // @author Tom Peters // -// @brief Converts protocol constant string to enum +// @brief Repository of state information shared by NHttpStreamSplitter and NHttpInspect classes // #ifndef NHTTP_FLOW_DATA_H @@ -61,6 +61,9 @@ private: // StreamSplitter internal data int64_t octets_seen[2] = { 0, 0 }; int num_crlf[2] = { 0, 0 }; + uint32_t peek_ahead_octets[2] = { 0, 0 }; + uint32_t unused_octets_visible[2] = { 0, 0 }; + uint32_t header_octets_visible[2] = { 0, 0 }; uint8_t *section_buffer[2] = { nullptr, nullptr }; int32_t section_buffer_length[2] = { 0, 0 }; uint8_t *chunk_buffer[2] = { nullptr, nullptr }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc index 6f61194bf..a3fbd5cf9 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc @@ -31,7 +31,6 @@ #include #include #include -#include #include "snort.h" #include "stream/stream_api.h" @@ -42,21 +41,10 @@ using namespace NHttpEnums; -NHttpInspect::NHttpInspect(bool test_input_, bool test_output_) : test_output(test_output_) +NHttpInspect::NHttpInspect(bool test_input, bool test_output) : test_manager(test_output) { - NHttpTestInput::test_input = test_input_; - if (NHttpTestInput::test_input) { - NHttpTestInput::test_input_source = new NHttpTestInput(test_input_file); - } -} - -NHttpInspect::~NHttpInspect () -{ - if (NHttpTestInput::test_input) { - delete NHttpTestInput::test_input_source; - if (test_out) { - fclose(test_out); - } + if (test_input) { + NHttpTestManager::activate_test_input(); } } @@ -139,20 +127,14 @@ ProcessResult NHttpInspect::process(const uint8_t* data, const uint16_t dsize, F msg_section->legacy_clients(); } - if (test_output) { - if (!NHttpTestInput::test_input) { + if (test_manager.use_test_output()) { + if (!NHttpTestManager::use_test_input()) { msg_section->print_section(stdout); } else { - if (NHttpTestInput::test_input_source->get_test_number() != file_test_number) { - if (test_out) fclose (test_out); - file_test_number = NHttpTestInput::test_input_source->get_test_number(); - char file_name[100]; - snprintf(file_name, sizeof(file_name), "%s%" PRIi64 ".txt", test_output_prefix, file_test_number); - if ((test_out = fopen(file_name, "w+")) == nullptr) throw std::runtime_error("Cannot open test output file"); - } - msg_section->print_section(test_out); - printf("Finished processing section from test %" PRIi64 "\n", file_test_number); + test_manager.update_test_number(NHttpTestManager::get_test_input_source()->get_test_number()); + msg_section->print_section(test_manager.get_output_file()); + printf("Finished processing section from test %" PRIi64 "\n", test_manager.get_test_number()); } fflush(nullptr); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h index 17f76bf9d..600470412 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h @@ -35,6 +35,7 @@ #include "nhttp_msg_chunk_head.h" #include "nhttp_msg_chunk_body.h" #include "nhttp_msg_trailer.h" +#include "nhttp_test_manager.h" #include "nhttp_stream_splitter.h" #include "nhttp_test_input.h" @@ -43,7 +44,6 @@ class NHttpApi; class NHttpInspect : public Inspector { public: NHttpInspect(bool test_input_, bool _test_output_); - ~NHttpInspect(); bool get_buf(InspectionBuffer::Type, Packet*, InspectionBuffer&); bool get_buf(unsigned, Packet*, InspectionBuffer&); @@ -63,13 +63,7 @@ private: NHttpEnums::ProcessResult process(const uint8_t* data, const uint16_t dsize, Flow* const flow, NHttpEnums::SourceId source_id_, bool buf_owner); - - // Test mode - bool test_output; - const char *test_input_file = "nhttp_test_msgs.txt"; - const char *test_output_prefix = "nhttpresults/testcase"; - FILE *test_out = nullptr; - int64_t file_test_number = -1; + NHttpTestManager test_manager; }; #endif diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc index aecd255e3..60b66aaf7 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc @@ -152,11 +152,6 @@ void NHttpMsgHeadShared::gen_events() { if (infractions & INF_TOOMANYHEADERS) create_event(EVENT_MAX_HEADERS); } -ProcessResult NHttpMsgHeadShared::worth_detection() { - // Do not send empty headers or trailers to detection - return (headers.length != STAT_NOTPRESENT) ? RES_INSPECT : RES_IGNORE; -} - void NHttpMsgHeadShared::print_headers(FILE *output) { char title_buf[100]; if (num_headers != STAT_NOSOURCE) fprintf(output, "Number of headers: %d\n", num_headers); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h index 62210e40f..94c58848e 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h @@ -42,7 +42,6 @@ class NHttpMsgHeadShared: public NHttpMsgSection { public: void analyze(); void gen_events(); - NHttpEnums::ProcessResult worth_detection(); int32_t get_num_headers() const { return num_headers; }; const Field& get_headers() const { return headers; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc index ce939ce7d..f735bd22c 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc @@ -104,6 +104,22 @@ void NHttpMsgHeader::update_flow() { } } +ProcessResult NHttpMsgHeader::worth_detection() { + // We can combine with body when sending to detection if the entire body is already available and the combined + // size does exceed paf_max. + if ((session_data->type_expected[source_id] == SEC_BODY) && + (session_data->data_length[source_id] <= session_data->unused_octets_visible[source_id]) && + (session_data->data_length[source_id] <= 16384) && + (session_data->section_buffer_length[source_id] + msg_text.length + session_data->data_length[source_id] <= 63780)) + { + return RES_AGGREGATE; + } + + // Do not send empty headers by themselves to detection + return ((headers.length != STAT_NOTPRESENT) || (session_data->section_buffer_length[source_id] > 0)) + ? RES_INSPECT : RES_IGNORE; +} + // Legacy support function. Puts message fields into the buffers used by old Snort. void NHttpMsgHeader::legacy_clients() { ClearHttpBuffers(); @@ -113,3 +129,9 @@ void NHttpMsgHeader::legacy_clients() { } + + + + + + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h index d0c599038..bcaf75f7a 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h @@ -42,6 +42,7 @@ public: void print_section(FILE *output); void gen_events(); void update_flow(); + NHttpEnums::ProcessResult worth_detection(); void legacy_clients(); }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc index 9af9703ae..c2c8bceae 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc @@ -84,6 +84,39 @@ void NHttpMsgStart::derive_version_id() { void NHttpMsgStart::gen_events() {} ProcessResult NHttpMsgStart::worth_detection() { - return RES_INSPECT; + // We combine the start line with the headers for sending to detection if they are already available and we will + // not exceed paf_max. + if ((session_data->header_octets_visible[source_id] > 0) && + (session_data->type_expected[source_id] == SEC_HEADER) && + (msg_text.length + session_data->header_octets_visible[source_id]) <= 63780) { + return RES_AGGREGATE; + } + else { + return RES_INSPECT; + } } + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc index a264585d1..ed4f7ce89 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc @@ -66,6 +66,11 @@ void NHttpMsgTrailer::update_flow() { } } +ProcessResult NHttpMsgTrailer::worth_detection() { + // Do not send empty trailers to detection + return (headers.length != STAT_NOTPRESENT) ? RES_INSPECT : RES_IGNORE; +} + // Legacy support function. Puts message fields into the buffers used by old Snort. void NHttpMsgTrailer::legacy_clients() { ClearHttpBuffers(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h index eb7871e56..3fc91486c 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h @@ -42,6 +42,7 @@ public: void print_section(FILE *output); void gen_events(); void update_flow(); + NHttpEnums::ProcessResult worth_detection(); void legacy_clients(); }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc index 0398dd282..179e7c6b1 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc @@ -32,6 +32,7 @@ #include "snort.h" #include "protocols/packet.h" #include "nhttp_enum.h" +#include "nhttp_test_manager.h" #include "nhttp_test_input.h" #include "nhttp_stream_splitter.h" #include "nhttp_inspect.h" @@ -40,24 +41,167 @@ using namespace NHttpEnums; // Convenience function. All the housekeeping that must be done before we can return FLUSH to stream. void NHttpStreamSplitter::prepare_flush(NHttpFlowData* session_data, uint32_t* flush_offset, SourceId source_id, - SectionType section_type, bool tcp_close, uint64_t infractions, uint32_t num_octets) { + SectionType section_type, bool tcp_close, uint64_t infractions, uint32_t num_octets, uint32_t length) { session_data->section_type[source_id] = section_type; session_data->tcp_close[source_id] = tcp_close; session_data->infractions[source_id] = infractions; if (tcp_close) { session_data->type_expected[source_id] = SEC_CLOSED; } - if (!NHttpTestInput::test_input) { + if (!NHttpTestManager::use_test_input()) { *flush_offset = num_octets; } else { - NHttpTestInput::test_input_source->flush(num_octets); + NHttpTestManager::get_test_input_source()->flush(num_octets); } session_data->octets_seen[source_id] = 0; session_data->num_crlf[source_id] = 0; + session_data->peek_ahead_octets[source_id] = 0; + session_data->unused_octets_visible[source_id] = length - num_octets; + session_data->header_octets_visible[source_id] = 0; } -const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total*/, unsigned offset, const uint8_t* data, +StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t length, uint32_t, uint32_t* flush_offset) { + // When the system begins providing TCP connection close information this won't always be false. FIXIT-H + bool tcp_close = false; + + // This is the session state information we share with HTTP Inspect and store with stream. A session is defined + // by a TCP connection. Since PAF is the first to see a new TCP connection the new flow data object is created here. + NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data(NHttpFlowData::nhttp_flow_id); + if (session_data == nullptr) flow->set_application_data(session_data = new NHttpFlowData); + assert(session_data != nullptr); + + SourceId source_id = to_server() ? SRC_CLIENT : SRC_SERVER; + + if (NHttpTestManager::use_test_input()) { + // This block substitutes a completely new data buffer supplied by the test tool in place of the "real" data. + // It also rewrites the buffer length, source ID, and TCP close indicator. + *flush_offset = length; + bool need_break; + uint8_t* test_data = nullptr; + NHttpTestManager::get_test_input_source()->scan(test_data, length, source_id, tcp_close, need_break); + if (length == 0) { + return StreamSplitter::FLUSH; + } + data = test_data; + if (need_break) { + session_data = new NHttpFlowData; + flow->set_application_data(session_data); + } + assert(session_data->type_expected[source_id] != SEC_ABORT); + assert(session_data->type_expected[source_id] != SEC_CLOSED); + } + + SectionType type = session_data->type_expected[source_id]; + + // Check for header section previously found during peek ahead + if ((type == SEC_HEADER) && (session_data->header_octets_visible[source_id] > 0)) { + prepare_flush(session_data, flush_offset, source_id, type, + tcp_close && (session_data->header_octets_visible[source_id] == length), + 0, session_data->peek_ahead_octets[source_id], length); + return StreamSplitter::FLUSH; + } + + switch (type) { + case SEC_REQUEST: + case SEC_STATUS: + case SEC_HEADER: + case SEC_CHUNKHEAD: + case SEC_TRAILER: + paf_max = 63780; + for (uint32_t k = session_data->peek_ahead_octets[source_id]; k < length; k++) { + session_data->octets_seen[source_id]++; + // Count the alternating and characters we have seen in a row + if (((data[k] == '\r') && (session_data->num_crlf[source_id]%2 == 0)) || + ((data[k] == '\n') && (session_data->num_crlf[source_id]%2 == 1))) { + session_data->num_crlf[source_id]++; + } + else { + session_data->num_crlf[source_id] = 0; + } + + // If the first two octets are CRLF then flush them separately. We are 1) DISCARDing CRLF some + // 1.0 implementation put following previous message, 2) DISCARDing CRLF between chunk and following + // chunk header, and 3) flushing normal empty header or trailer. + if ((session_data->num_crlf[source_id] == 2) && (session_data->octets_seen[source_id] == 2)) { + prepare_flush(session_data, flush_offset, source_id, + ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD)) ? SEC_DISCARD : type, + tcp_close && (k == length-1), 0, k+1, length); + return StreamSplitter::FLUSH; + } + // The start line and chunk header section always end with the first + else if ((session_data->num_crlf[source_id] == 2) && + ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD))) { + prepare_flush(session_data, flush_offset, source_id, type, tcp_close && (k == length-1), 0, k+1, length); + if ((type == SEC_REQUEST) || (type == SEC_STATUS)) { + // Look ahead to see if entire header section is already here so we can aggregate it for detection. + for (uint32_t m = k+1; m < length; m++) { + session_data->octets_seen[source_id]++; + // Count the alternating and characters we have seen in a row + if (((data[m] == '\r') && (session_data->num_crlf[source_id]%2 == 0)) || + ((data[m] == '\n') && (session_data->num_crlf[source_id]%2 == 1))) { + session_data->num_crlf[source_id]++; + } + else { + session_data->num_crlf[source_id] = 0; + } + if ( (session_data->num_crlf[source_id] == 4) || + ((session_data->num_crlf[source_id] == 2) && (session_data->octets_seen[source_id] == 2))) { + session_data->header_octets_visible[source_id] = m-k; + return StreamSplitter::FLUSH; + } + } + session_data->peek_ahead_octets[source_id] = length - (k+1); + } + return StreamSplitter::FLUSH; + } + // The header and trailer sections always end with the first double + else if (session_data->num_crlf[source_id] == 4) { + prepare_flush(session_data, flush_offset, source_id, type, tcp_close && (k == length-1), 0, k+1, length); + return StreamSplitter::FLUSH; + } + // We must do this to protect ourself from buffer overrun. + else if (session_data->octets_seen[source_id] >= 63780) { + // FIXIT-M need to implement processing and detection instead of just discarding this data + session_data->type_expected[source_id] = SEC_ABORT; + return StreamSplitter::ABORT; + } + } + session_data->peek_ahead_octets[source_id] = 0; + // Incomplete headers wait patiently for more data + if (!tcp_close) { + return StreamSplitter::SEARCH; + } + // Discard the oddball case where the new "message" starts with + else if ((session_data->octets_seen[source_id] == 1) && (session_data->num_crlf[source_id] == 1)) { + prepare_flush(session_data, flush_offset, source_id, SEC_DISCARD, true, 0, length, length); + } + // TCP connection close, flush the partial header + else { + prepare_flush(session_data, flush_offset, source_id, type, true, INF_TRUNCATED, length, length); + } + return StreamSplitter::FLUSH; + case SEC_BODY: + case SEC_CHUNKBODY: + paf_max = 16384 - session_data->chunk_buffer_length[source_id]; + if ((!tcp_close) || (length > session_data->data_length[source_id])) { + prepare_flush(session_data, flush_offset, source_id, type, false, 0, session_data->data_length[source_id], + length); + } + else { + // The TCP connection has closed and this is the possibly incomplete final section + prepare_flush(session_data, flush_offset, source_id, type, true, 0, length, length); + } + return StreamSplitter::FLUSH; + case SEC_ABORT: + return StreamSplitter::ABORT; + default: + assert(0); + return StreamSplitter::ABORT; + } +} + +const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total FIXIT-H */, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) { static THREAD_LOCAL StreamBuffer nhttp_buf; @@ -66,13 +210,13 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total SourceId source_id = to_server() ? SRC_CLIENT : SRC_SERVER; copied = len; - if (NHttpTestInput::test_input) { + if (NHttpTestManager::use_test_input()) { if (!(flags & PKT_PDU_TAIL)) { return nullptr; } uint8_t* test_buffer; - NHttpTestInput::test_input_source->reassemble(&test_buffer, len, source_id, session_data); + NHttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, source_id, session_data); if (len == 0) { // There is no more test data return nullptr; @@ -128,6 +272,10 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total nhttp_buf.length = buffer_length + offset + len; buffer = nullptr; buffer_length = 0; + if (my_inspector->test_manager.use_test_output()) { + FILE* out_file = NHttpTestManager::use_test_input() ? my_inspector->test_manager.get_output_file() : stdout; + fprintf(out_file, "Sent to detection %u octets\n\n", nhttp_buf.length); + } return &nhttp_buf; case RES_IGNORE: buffer = nullptr; @@ -150,6 +298,10 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total nhttp_buf.length = chunk_buffer_length; chunk_buffer = nullptr; chunk_buffer_length = 0; + if (my_inspector->test_manager.use_test_output()) { + FILE* out_file = NHttpTestManager::use_test_input() ? my_inspector->test_manager.get_output_file() : stdout; + fprintf(out_file, "Flushed chunks for detection %u octets\n\n", nhttp_buf.length); + } return &nhttp_buf; } return nullptr; @@ -158,114 +310,6 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total return nullptr; } -StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t length, uint32_t, uint32_t* flush_offset) { - // When the system begins providing TCP connection close information this won't always be false. FIXIT-H - bool tcp_close = false; - - // This is the session state information we share with HTTP Inspect and store with stream. A session is defined - // by a TCP connection. Since PAF is the first to see a new TCP connection the new flow data object is created here. - NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data(NHttpFlowData::nhttp_flow_id); - if (session_data == nullptr) flow->set_application_data(session_data = new NHttpFlowData); - assert(session_data != nullptr); - - SourceId source_id = to_server() ? SRC_CLIENT : SRC_SERVER; - - if (NHttpTestInput::test_input) { - // This block substitutes a completely new data buffer supplied by the test tool in place of the "real" data. - // It also rewrites the buffer length, source ID, and TCP close indicator. - *flush_offset = length; - bool need_break; - uint8_t* test_data = nullptr; - NHttpTestInput::test_input_source->scan(test_data, length, source_id, tcp_close, need_break); - if (length == 0) { - return StreamSplitter::FLUSH; - } - data = test_data; - if (need_break) { - flow->set_application_data(session_data = new NHttpFlowData); - } - } - - switch (SectionType type = session_data->type_expected[source_id]) { - case SEC_REQUEST: - case SEC_STATUS: - case SEC_HEADER: - case SEC_CHUNKHEAD: - case SEC_TRAILER: - paf_max = 63780; - for (uint32_t k = 0; k < length; k++) { - session_data->octets_seen[source_id]++; - // Count the alternating and characters we have seen in a row - if (((data[k] == '\r') && (session_data->num_crlf[source_id]%2 == 0)) || - ((data[k] == '\n') && (session_data->num_crlf[source_id]%2 == 1))) { - session_data->num_crlf[source_id]++; - } - else { - session_data->num_crlf[source_id] = 0; - } - - // If the first two octets are CRLF then flush them separately. We are 1) DISCARDing CRLF some - // 1.0 implementation put following previous message, 2) DISCARDing CRLF between chunk and following - // chunk header, and 3) flushing normal empty header or trailer. - if ((session_data->num_crlf[source_id] == 2) && (session_data->octets_seen[source_id] == 2)) { - prepare_flush(session_data, flush_offset, source_id, - ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD)) ? SEC_DISCARD : type, - tcp_close && (k == length-1), 0, k+1); - return StreamSplitter::FLUSH; - } - // The start line and chunk header section always end with the first - else if ((session_data->num_crlf[source_id] == 2) && - ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD))) { - prepare_flush(session_data, flush_offset, source_id, type, tcp_close && (k == length-1), 0, k+1); - return StreamSplitter::FLUSH; - } - // The header and trailer sections always end with the first double - else if (session_data->num_crlf[source_id] == 4) { - prepare_flush(session_data, flush_offset, source_id, type, tcp_close && (k == length-1), 0, k+1); - return StreamSplitter::FLUSH; - } - // We must do this to protect ourself from buffer overrun. - else if (session_data->octets_seen[source_id] >= 63780) { - prepare_flush(session_data, flush_offset, source_id, type, tcp_close && (k == length-1), INF_HEADTOOLONG, k+1); - return StreamSplitter::FLUSH; - } - } - // Incomplete headers wait patiently for more data - if (!tcp_close) { - return StreamSplitter::SEARCH; - } - // Discard the oddball case where the new "message" starts with - else if ((session_data->octets_seen[source_id] == 1) && (session_data->num_crlf[source_id] == 1)) { - prepare_flush(session_data, flush_offset, source_id, SEC_DISCARD, true, 0, length); - } - // TCP connection close, flush the partial header - else { - prepare_flush(session_data, flush_offset, source_id, type, true, INF_TRUNCATED, length); - } - return StreamSplitter::FLUSH; - case SEC_BODY: - case SEC_CHUNKBODY: - paf_max = 16384 - session_data->chunk_buffer_length[source_id]; - if ((!tcp_close) || (length > session_data->data_length[source_id])) { - prepare_flush(session_data, flush_offset, source_id, type, false, 0, session_data->data_length[source_id]); - } - else { - // The TCP connection has closed and this is the possibly incomplete final section - prepare_flush(session_data, flush_offset, source_id, type, true, 0, length); - } - return StreamSplitter::FLUSH; - case SEC_ABORT: - return StreamSplitter::ABORT; - default: - assert(0); - return StreamSplitter::ABORT; - } -} - - - - - diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h index ee89d1936..7cb89d5c2 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h @@ -31,7 +31,7 @@ #include "stream/stream_splitter.h" #include "nhttp_flow_data.h" -#include "nhttp_test_input.h" +#include "nhttp_test_manager.h" class NHttpInspect; @@ -43,10 +43,10 @@ public: const StreamBuffer* reassemble(Flow* flow, unsigned total, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied); bool is_paf() { return true; }; - uint32_t max() { return NHttpTestInput::test_input ? 16384 : paf_max; }; + uint32_t max() { return NHttpTestManager::use_test_input() ? 16384 : paf_max; }; private: void prepare_flush(NHttpFlowData* session_data, uint32_t* flush_offset, NHttpEnums::SourceId source_id, - NHttpEnums::SectionType section_type, bool tcp_close, uint64_t infractions, uint32_t num_octets); + NHttpEnums::SectionType section_type, bool tcp_close, uint64_t infractions, uint32_t num_octets, uint32_t length); void create_event(NHttpEnums::EventSid sid); NHttpInspect* const my_inspector; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc index ced08f912..fa7ea44a4 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc @@ -33,14 +33,10 @@ #include #include -#include "nhttp_enum.h" #include "nhttp_test_input.h" using namespace NHttpEnums; -bool NHttpTestInput::test_input = false; -NHttpTestInput *NHttpTestInput::test_input_source = nullptr; - NHttpTestInput::NHttpTestInput(const char *file_name) { if ((test_data_file = fopen(file_name, "r")) == nullptr) throw std::runtime_error("Cannot open test input file"); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h index 626eaed5d..7b6227fc6 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h @@ -29,6 +29,7 @@ #ifndef NHTTP_TEST_INPUT_H #define NHTTP_TEST_INPUT_H +#include "nhttp_enum.h" #include "nhttp_flow_data.h" class NHttpTestInput { @@ -39,8 +40,6 @@ public: void flush(uint32_t length); void reassemble(uint8_t **buffer, unsigned &length, NHttpEnums::SourceId &source_id, NHttpFlowData* session_data); - static bool test_input; - static NHttpTestInput *test_input_source; int64_t get_test_number() { return test_number; }; private: FILE *test_data_file; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc new file mode 100644 index 000000000..c31e00688 --- /dev/null +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.cc @@ -0,0 +1,46 @@ +/**************************************************************************** + * +** Copyright (C) 2014 Cisco and/or its affiliates. All rights reserved. + * Copyright (C) 2003-2013 Sourcefire, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License Version 2 as + * published by the Free Software Foundation. You may not use, modify or + * distribute this program under any other version of the GNU General + * Public License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + ****************************************************************************/ + +// +// @author Tom Peters +// +// @brief Control NHttpInspect test tools +// + +#include +#include "nhttp_test_manager.h" +#include "nhttp_test_input.h" + +bool NHttpTestManager::test_input = false; +NHttpTestInput NHttpTestManager::test_input_source("nhttp_test_msgs.txt"); +const char* NHttpTestManager::test_output_prefix = "nhttpresults/testcase"; + +void NHttpTestManager::update_test_number(int64_t new_test_number) { + if (new_test_number != test_number) { + if (test_out != nullptr) fclose (test_out); + test_number = new_test_number; + char file_name[100]; + snprintf(file_name, sizeof(file_name), "%s%" PRIi64 ".txt", test_output_prefix, test_number); + if ((test_out = fopen(file_name, "w+")) == nullptr) throw std::runtime_error("Cannot open test output file"); + } +} + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h new file mode 100644 index 000000000..40f1975ad --- /dev/null +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_manager.h @@ -0,0 +1,89 @@ +/**************************************************************************** + * +** Copyright (C) 2014 Cisco and/or its affiliates. All rights reserved. + * Copyright (C) 2003-2013 Sourcefire, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License Version 2 as + * published by the Free Software Foundation. You may not use, modify or + * distribute this program under any other version of the GNU General + * Public License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + ****************************************************************************/ + +// +// @author Tom Peters +// +// @brief NHttpTestManager class declaration +// + +#ifndef NHTTP_TEST_MANAGER_H +#define NHTTP_TEST_MANAGER_H + +#include +#include +#include + +//------------------------------------------------------------------------- +// NHttpTestManager class +//------------------------------------------------------------------------- + +class NHttpTestInput; + +class NHttpTestManager { + friend class NHttpStreamSplitter; + friend class NHttpInspect; + +public: + static bool use_test_input() { return test_input; }; + static void activate_test_input() { test_input = true; }; + static NHttpTestInput *get_test_input_source() { return &test_input_source; }; + + NHttpTestManager(bool test_output_) : test_output(test_output_) {}; + ~NHttpTestManager() { if (test_out != nullptr) fclose(test_out); }; + bool use_test_output() const { return test_output; }; + void update_test_number(int64_t new_test_number); + FILE* get_output_file() { assert(test_out != nullptr); return test_out; }; + int64_t get_test_number() const { return test_number; }; + +private: + // Test input read from file + static bool test_input; + static NHttpTestInput test_input_source; + + // Printing results of message processing + const bool test_output; + static const char* test_output_prefix; + FILE* test_out = nullptr; + int64_t test_number = -1; +}; + +#endif + + + + + + + + + + + + + + + + + + + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt index 3ac973d90..54f9dd425 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt @@ -14,6 +14,7 @@ # # Command lines are left justified, lower case, with no whitespace: # @break resets HTTP Inspect data structures and begins a new test. Use it liberally to prevent unrelated tests from interfering with each other. +# @tcpclose simulates a half-duplex TCP close following the next paragraph of data. # @request and @response set the message direction. Applies to subsequent sections until changed. # @ sets the test number and hence the test output file name. Applies to subsequent sections until changed. Don't reuse numbers. # @@ -32,6 +33,15 @@ # Content-Length field or chunk header. Specifically flushing more data than in the current segment will trigger filling. The user should include at # least one character of body/chunk data either as part of the previous header segment or at the beginning of a new segment following the headers. # All data bytes included in the file will be used followed by required fill data in the pattern ABC...XYZABC... +# +# There must not be excess data for a test case. Once a data stream ends with a TCP close there must be a break command before further data is sent. +# Similarly a message section that triggers an abort of processing must end the paragraph and be followed by a break. These rules apply half-duplex +# so it would be possible to send data in the opposite direction before the break. +# +# FIXIT-L it would be useful if a paragraph could continue following an abort to simulate a processing abort mid-segment. +# +# This test tool does not implement the feature of being hardened against bad input. If you write a badly formatted or improper test case the +# program may assert or crash. The responsibility is on the developer to get it right. Currently that is the best use of resources. # *********************************************************************************************** @@ -78,12 +88,12 @@ HTTP/1.1 301 \r\n\r\n @response HTTP/1.1 560 \r\n\r\n -@1010 +@1009 @break @response HTTP/1.1 111 \r\n\r\n -@1011 +@1010 @break @response HTTP/1.1 234 qwertyuiopasdfghjklzxcvbnmABCDEFGHIJKLMNOPQRSTUVWXYZ qwertyuiopasdfghjklzxcvbnmABCDEFGHIJKLMNOPQRSTUVWXYZqwertyuiopasdfghjklzxcvbnmABCDE{}[]