From: Tom Peters Date: Mon, 8 Sep 2014 15:34:50 +0000 (-0400) Subject: chunk aggregation complete X-Git-Tag: 3.0.0-233~1418 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=96580c94920cdaeb1508736d76ed68a283ea5b39;p=thirdparty%2Fsnort3.git chunk aggregation complete --- diff --git a/src/service_inspectors/nhttp_inspect/nhttp_enum.h b/src/service_inspectors/nhttp_inspect/nhttp_enum.h index 758d9a883..32fd89edb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_enum.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_enum.h @@ -49,6 +49,9 @@ typedef enum { SRC__NOTCOMPUTE=-4, SRC_CLIENT=0, SRC_SERVER=1 } SourceId; typedef enum { SEC__NOTCOMPUTE=-4, SEC__NOTPRESENT=-1, SEC_REQUEST = 2, SEC_STATUS, SEC_HEADER, SEC_BODY, SEC_CHUNKHEAD, SEC_CHUNKBODY, SEC_TRAILER, SEC_DISCARD, SEC_CLOSED, SEC_ABORT } SectionType; +// Result of processing a message section--what needs to happen next +typedef enum { RES_INSPECT, RES_IGNORE, RES_AGGREGATE, RES_FLUSHCHUNKS } ProcessResult; + // List of possible HTTP versions. Version 0.9 omitted because 0.9 predates creation of the HTTP/X.Y token. There would // never be a message with "HTTP/0.9" typedef enum { VERS__NOSOURCE=-6, VERS__NOTCOMPUTE=-4, VERS__PROBLEMATIC=-2, VERS__NOTPRESENT=-1, VERS__OTHER=1, VERS_1_0, @@ -87,7 +90,7 @@ typedef enum { HEAD__NOTCOMPUTE=-4, HEAD__INSUFMEMORY=-3, HEAD__PROBLEMATIC=-2, typedef enum { INF_TRUNCATED=0x1, INF_HEADTOOLONG=0x2, INF_BADREQLINE=0x4, INF_BADSTATLINE=0x8, INF_TOOMANYHEADERS=0x10, INF_BADHEADER=0x20, INF_BADSTATCODE=0x40, INF_UNKNOWNVERSION=0x80, INF_BADVERSION=0x100, INF_NOSCRATCH=0x200, - INF_BADHEADERREPS=0x400, INF_BADHEADERDATA=0x800, INF_BROKENCHUNK=0x1000, INF_BADCHUNKSIZE=0x2000, + INF_BADHEADERREPS=0x400, INF_BADHEADERDATA=0x800, INF_BADCHUNKSIZE=0x2000, INF_BADPHRASE=0x4000, INF_BADURI=0x8000, INF_BADPORT=0x10000, INF_URINEEDNORM=0x20000, INF_URIPERCENTNORMAL=0x40000, INF_URIPERCENTASCII=0x80000, INF_URIPERCENTUTF8=0x100000, INF_URIPERCENTUCODE=0x200000, INF_URIPERCENTOTHER=0x400000, INF_URIBADCHAR=0x800000, INF_URI8BITCHAR=0x1000000, INF_URIMULTISLASH=0x2000000, INF_URIBACKSLASH=0x4000000, diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc index 4f6d7472c..7539e4267 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc @@ -41,25 +41,24 @@ unsigned NHttpFlowData::nhttp_flow_id = 0; NHttpFlowData::NHttpFlowData() : FlowData(nhttp_flow_id) { } NHttpFlowData::~NHttpFlowData() { - delete transaction[SRC_CLIENT]; - delete transaction[SRC_SERVER]; + for (int k=0; k <= 1; k++) { + delete[] section_buffer[k]; + delete[] chunk_buffer[k]; + delete transaction[k]; + } delete_pipeline(); } void NHttpFlowData::half_reset(SourceId source_id) { assert((source_id == SRC_CLIENT) || (source_id == SRC_SERVER)); - octets_expected[source_id] = STAT_NOTPRESENT; version_id[source_id] = VERS__NOTPRESENT; method_id[source_id] = METH__NOTPRESENT; status_code_num[source_id] = STAT_NOTPRESENT; data_length[source_id] = STAT_NOTPRESENT; - body_sections[source_id] = STAT_NOTPRESENT; body_octets[source_id] = STAT_NOTPRESENT; num_chunks[source_id] = STAT_NOTPRESENT; - chunk_sections[source_id] = STAT_NOTPRESENT; - chunk_octets[source_id] = STAT_NOTPRESENT; } bool NHttpFlowData::add_to_pipeline(NHttpTransaction* latest) { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h index d0293eddb..023738a5f 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h @@ -54,12 +54,17 @@ public: friend class NHttpMsgTrailer; friend class NHttpStreamSplitter; friend class NHttpTransaction; + friend class NHttpTestInput; private: void half_reset(NHttpEnums::SourceId source_id); // StreamSplitter internal data int64_t octets_seen[2] = { 0, 0 }; int num_crlf[2] = { 0, 0 }; + uint8_t *section_buffer[2] = { nullptr, nullptr }; + int32_t section_buffer_length[2] = { 0, 0 }; + uint8_t *chunk_buffer[2] = { nullptr, nullptr }; + int32_t chunk_buffer_length[2] = { 0, 0 }; // StreamSplitter => Inspector (facts about the most recent message section) // 0 element refers to client request, 1 element refers to server response @@ -69,7 +74,7 @@ private: // Inspector => StreamSplitter (facts about the message section that is coming next) NHttpEnums::SectionType type_expected[2] = { NHttpEnums::SEC_REQUEST, NHttpEnums::SEC_STATUS }; - int64_t octets_expected[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // expected size of the upcoming body or chunk body section + int64_t data_length[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // length of the data from Content-Length field or chunk header. // Inspector's internal data about the current message // Some items don't apply in both directions. Have two copies anyway just to simplify code and minimize @@ -78,12 +83,8 @@ private: NHttpEnums::MethodId method_id[2] = { NHttpEnums::METH__NOTPRESENT, NHttpEnums::METH__NOTPRESENT }; int32_t status_code_num[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; - int64_t data_length[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // length of the data from Content-Length field or chunk header. - int64_t body_sections[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of body sections seen so far including chunk headers - int64_t body_octets[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of user data octets seen so far (either regular body or chunks) - int64_t num_chunks[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of chunks seen so far - int64_t chunk_sections[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of sections seen so far in the current chunk - int64_t chunk_octets[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of user data octets seen so far in the current chunk including terminating CRLF + int64_t body_octets[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of user data octets seen so far (regular body or chunks) + int64_t num_chunks[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of chunks seen so far // Transaction management including pipelining NHttpTransaction* transaction[2] = { nullptr, nullptr }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc index e4a2cdb3a..6f61194bf 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc @@ -111,7 +111,7 @@ void NHttpInspect::show(SnortConfig*) LogMessage("NHttpInspect\n"); } -void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* const flow, SourceId source_id) +ProcessResult NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* const flow, SourceId source_id, bool buf_owner) { NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data(NHttpFlowData::nhttp_flow_id); assert(session_data); @@ -119,21 +119,25 @@ void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* cons NHttpMsgSection *msg_section = nullptr; switch (session_data->section_type[source_id]) { - case SEC_REQUEST: msg_section = new NHttpMsgRequest(data, dsize, session_data, source_id); break; - case SEC_STATUS: msg_section = new NHttpMsgStatus(data, dsize, session_data, source_id); break; - case SEC_HEADER: msg_section = new NHttpMsgHeader(data, dsize, session_data, source_id); break; - case SEC_BODY: msg_section = new NHttpMsgBody(data, dsize, session_data, source_id); break; - case SEC_CHUNKHEAD: msg_section = new NHttpMsgChunkHead(data, dsize, session_data, source_id); break; - case SEC_CHUNKBODY: msg_section = new NHttpMsgChunkBody(data, dsize, session_data, source_id); break; - case SEC_TRAILER: msg_section = new NHttpMsgTrailer(data, dsize, session_data, source_id); break; - case SEC_DISCARD: delete[] data; return; - default: assert(0); delete[] data; return; + case SEC_REQUEST: msg_section = new NHttpMsgRequest(data, dsize, session_data, source_id, buf_owner); break; + case SEC_STATUS: msg_section = new NHttpMsgStatus(data, dsize, session_data, source_id, buf_owner); break; + case SEC_HEADER: msg_section = new NHttpMsgHeader(data, dsize, session_data, source_id, buf_owner); break; + case SEC_BODY: msg_section = new NHttpMsgBody(data, dsize, session_data, source_id, buf_owner); break; + case SEC_CHUNKHEAD: msg_section = new NHttpMsgChunkHead(data, dsize, session_data, source_id, buf_owner); break; + case SEC_CHUNKBODY: msg_section = new NHttpMsgChunkBody(data, dsize, session_data, source_id, buf_owner); break; + case SEC_TRAILER: msg_section = new NHttpMsgTrailer(data, dsize, session_data, source_id, buf_owner); break; + case SEC_DISCARD: if (buf_owner) delete[] data; return RES_IGNORE; + default: assert(0); if (buf_owner) delete[] data; return RES_IGNORE; } msg_section->analyze(); msg_section->update_flow(); msg_section->gen_events(); - msg_section->legacy_clients(); + + ProcessResult return_value = msg_section->worth_detection(); + if (return_value == RES_INSPECT) { + msg_section->legacy_clients(); + } if (test_output) { if (!NHttpTestInput::test_input) { @@ -152,8 +156,7 @@ void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* cons } fflush(nullptr); } -} - - + return return_value; +} diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h index 5d112526f..17f76bf9d 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h @@ -61,7 +61,8 @@ private: friend NHttpApi; friend NHttpStreamSplitter; - void process(const uint8_t* data, const uint16_t dsize, Flow* const flow, NHttpEnums::SourceId source_id_); + NHttpEnums::ProcessResult process(const uint8_t* data, const uint16_t dsize, Flow* const flow, + NHttpEnums::SourceId source_id_, bool buf_owner); // Test mode bool test_output; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc index 55c869391..700a136b1 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc @@ -38,21 +38,19 @@ using namespace NHttpEnums; -NHttpMsgBody::NHttpMsgBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgSection(buffer, buf_size, session_data_, source_id_), data_length(session_data->data_length[source_id]), - body_sections(session_data->body_sections[source_id]), body_octets(session_data->body_octets[source_id]) +NHttpMsgBody::NHttpMsgBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgSection(buffer, buf_size, session_data_, source_id_, buf_owner), + data_length(session_data->data_length[source_id]), body_octets(session_data->body_octets[source_id]) { transaction->set_body(this); } void NHttpMsgBody::analyze() { - body_sections++; body_octets += msg_text.length; data.start = msg_text.start; data.length = msg_text.length; - // The following statement tests for the case where streams underfulfilled flush due to a TCP connection close - if ((msg_text.length < 16384) && (body_octets < data_length)) tcp_close = true; if (tcp_close && (body_octets < data_length)) infractions |= INF_TRUNCATED; } @@ -61,7 +59,7 @@ void NHttpMsgBody::gen_events() { void NHttpMsgBody::print_section(FILE *output) { NHttpMsgSection::print_message_title(output, "body"); - fprintf(output, "Expected data length %" PRIi64 ", sections seen %" PRIi64 ", octets seen %" PRIi64 "\n", data_length, body_sections, body_octets); + fprintf(output, "Expected data length %" PRIi64 ", octets seen %" PRIi64 "\n", data_length, body_octets); data.print(output, "Data"); NHttpMsgSection::print_message_wrapup(output); } @@ -73,7 +71,6 @@ void NHttpMsgBody::update_flow() { } else if (body_octets < data_length) { // More body coming - session_data->body_sections[source_id] = body_sections; session_data->body_octets[source_id] = body_octets; } else { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.h index c003b9c52..c2a88d589 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.h @@ -38,7 +38,8 @@ class NHttpMsgBody : public NHttpMsgSection { public: - NHttpMsgBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void analyze(); void print_section(FILE *output); void gen_events(); @@ -47,7 +48,6 @@ public: protected: int64_t data_length; - int64_t body_sections; int64_t body_octets; Field data; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc index 3d93f53c7..9755a29bb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc @@ -38,37 +38,18 @@ using namespace NHttpEnums; -NHttpMsgChunkBody::NHttpMsgChunkBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgBody(buffer, buf_size, session_data_, source_id_), /* num_chunks(session_data->num_chunks[source_id]), &&& */ - chunk_sections(session_data->chunk_sections[source_id]), chunk_octets(session_data->chunk_octets[source_id]) +NHttpMsgChunkBody::NHttpMsgChunkBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : NHttpMsgBody(buffer, buf_size, session_data_, source_id_, buf_owner) { transaction->set_body(this); } void NHttpMsgChunkBody::analyze() { - body_sections++; - chunk_octets += msg_text.length; body_octets += msg_text.length; - int term_crlf_bytes = 0; - if (chunk_octets > data_length) { - // Final are not data and do not belong in octet total or data field - term_crlf_bytes = chunk_octets - data_length; - assert(term_crlf_bytes <= 2); - body_octets -= term_crlf_bytes; - // Check for correct CRLF termination. Beware the section might break just before chunk end. - if ( ! ( ((term_crlf_bytes == 2) && (msg_text.length >= 2) && (msg_text.start[msg_text.length-2] == '\r') && (msg_text.start[msg_text.length-1] == '\n')) || - ((term_crlf_bytes == 2) && (msg_text.length == 1) && (msg_text.start[msg_text.length-1] == '\n')) || - ((term_crlf_bytes == 1) && (msg_text.start[msg_text.length-1] == '\r')) ) ) { - infractions |= INF_BROKENCHUNK; - } - } data.start = msg_text.start; - data.length = msg_text.length - term_crlf_bytes; + data.length = msg_text.length; - chunk_sections++; - // The following statement tests for the case where streams underfulfilled flush due to a TCP connection close - if ((msg_text.length < 16384) && (body_octets + term_crlf_bytes < data_length + 2)) tcp_close = true; if (tcp_close) infractions |= INF_TRUNCATED; } @@ -77,8 +58,7 @@ void NHttpMsgChunkBody::gen_events() {} void NHttpMsgChunkBody::print_section(FILE *output) { NHttpMsgSection::print_message_title(output, "chunk body"); - fprintf(output, "Expected chunk length %" PRIi64 ", cumulative sections %" PRIi64 ", cumulative octets %" PRIi64 "\n", data_length, body_sections, body_octets); - fprintf(output, "cumulative chunk sections %" PRIi64 ", cumulative chunk octets %" PRIi64 "\n", chunk_sections, chunk_octets); + fprintf(output, "Cumulative octets %" PRIi64 "\n", body_octets); data.print(output, "Data"); NHttpMsgSection::print_message_wrapup(output); } @@ -88,20 +68,8 @@ void NHttpMsgChunkBody::update_flow() { session_data->type_expected[source_id] = SEC_CLOSED; session_data->half_reset(source_id); } - else if (chunk_octets < data_length + 2) { - session_data->body_sections[source_id] = body_sections; - session_data->body_octets[source_id] = body_octets; - session_data->chunk_sections[source_id] = chunk_sections; - session_data->chunk_octets[source_id] = chunk_octets; - } else { - session_data->type_expected[source_id] = SEC_CHUNKHEAD; - session_data->octets_expected[source_id] = STAT_NOTPRESENT; - session_data->data_length[source_id] = STAT_NOTPRESENT; - session_data->body_sections[source_id] = body_sections; session_data->body_octets[source_id] = body_octets; - session_data->chunk_sections[source_id] = STAT_NOTPRESENT; - session_data->chunk_octets[source_id] = STAT_NOTPRESENT; } } @@ -109,32 +77,3 @@ void NHttpMsgChunkBody::update_flow() { - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.h index 18d462b8c..06de6e841 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.h @@ -37,16 +37,12 @@ class NHttpMsgChunkBody : public NHttpMsgBody { public: - NHttpMsgChunkBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgChunkBody(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void analyze(); void print_section(FILE *output); void gen_events(); void update_flow(); - -private: - // int64_t num_chunks; // will be needed in future commented out to please compiler &&& - int64_t chunk_sections; - int64_t chunk_octets; }; #endif diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc index 1168370cb..ac00a6179 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc @@ -38,8 +38,9 @@ using namespace NHttpEnums; -NHttpMsgChunkHead::NHttpMsgChunkHead(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgSection(buffer, buf_size, session_data_, source_id_), body_sections(session_data->body_sections[source_id]), +NHttpMsgChunkHead::NHttpMsgChunkHead(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgSection(buffer, buf_size, session_data_, source_id_, buf_owner), num_chunks(session_data->num_chunks[source_id]) { transaction->set_body(this); @@ -79,7 +80,6 @@ void NHttpMsgChunkHead::derive_chunk_length() { } void NHttpMsgChunkHead::analyze() { - body_sections++; // First section in a new chunk is just the start line. num_chunks++; start_line.start = msg_text.start; @@ -122,19 +122,18 @@ void NHttpMsgChunkHead::update_flow() { } else if (data_length > 0) { session_data->type_expected[source_id] = SEC_CHUNKBODY; - session_data->octets_expected[source_id] = data_length+2; - session_data->body_sections[source_id] = body_sections; session_data->num_chunks[source_id] = num_chunks; session_data->data_length[source_id] = data_length; - session_data->chunk_sections[source_id] = 0; - session_data->chunk_octets[source_id] = 0; } - else { + else { // FIXIT-H what if data_length is bad (probable loss of sync)? // This was zero-length last chunk, trailer comes next session_data->type_expected[source_id] = SEC_TRAILER; } } +ProcessResult NHttpMsgChunkHead::worth_detection() { + return ((data_length > 0) && !tcp_close) ? RES_IGNORE : RES_FLUSHCHUNKS; +} // Legacy support function. Puts message fields into the buffers used by old Snort. void NHttpMsgChunkHead::legacy_clients() { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.h index 2a60d2832..606b9fb41 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.h @@ -38,12 +38,14 @@ class NHttpMsgChunkHead : public NHttpMsgSection { public: - NHttpMsgChunkHead(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgChunkHead(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void analyze(); void print_section(FILE *output); void gen_events(); void update_flow(); void legacy_clients(); + NHttpEnums::ProcessResult worth_detection(); private: void derive_chunk_length(); @@ -53,7 +55,6 @@ private: Field chunk_extensions; int64_t data_length = NHttpEnums::STAT_NOTCOMPUTE; - int64_t body_sections; int64_t num_chunks; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc index 60b66aaf7..aecd255e3 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.cc @@ -152,6 +152,11 @@ void NHttpMsgHeadShared::gen_events() { if (infractions & INF_TOOMANYHEADERS) create_event(EVENT_MAX_HEADERS); } +ProcessResult NHttpMsgHeadShared::worth_detection() { + // Do not send empty headers or trailers to detection + return (headers.length != STAT_NOTPRESENT) ? RES_INSPECT : RES_IGNORE; +} + void NHttpMsgHeadShared::print_headers(FILE *output) { char title_buf[100]; if (num_headers != STAT_NOSOURCE) fprintf(output, "Number of headers: %d\n", num_headers); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h index 0cdf8e3d4..62210e40f 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_head_shared.h @@ -42,6 +42,7 @@ class NHttpMsgHeadShared: public NHttpMsgSection { public: void analyze(); void gen_events(); + NHttpEnums::ProcessResult worth_detection(); int32_t get_num_headers() const { return num_headers; }; const Field& get_headers() const { return headers; }; @@ -53,7 +54,8 @@ public: protected: NHttpMsgHeadShared(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, - NHttpEnums::SourceId source_id_) : NHttpMsgSection(buffer, buf_size, session_data_, source_id_) {}; + NHttpEnums::SourceId source_id_, bool buf_owner) : + NHttpMsgSection(buffer, buf_size, session_data_, source_id_, buf_owner) {}; // Header normalization strategies. There should be one of these for every different way we can process // a header field value. diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc index ef674f50b..ce939ce7d 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.cc @@ -39,8 +39,9 @@ using namespace NHttpEnums; -NHttpMsgHeader::NHttpMsgHeader(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgHeadShared(buffer, buf_size, session_data_, source_id_) +NHttpMsgHeader::NHttpMsgHeader(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgHeadShared(buffer, buf_size, session_data_, source_id_, buf_owner) { transaction->set_header(this, source_id); } @@ -86,7 +87,6 @@ void NHttpMsgHeader::update_flow() { (get_header_value_norm(HEAD_TRANSFER_ENCODING).length - 8))) == TRANSCODE_CHUNKED) ) { // Chunked body session_data->type_expected[source_id] = SEC_CHUNKHEAD; - session_data->body_sections[source_id] = 0; session_data->body_octets[source_id] = 0; session_data->num_chunks[source_id] = 0; } @@ -94,9 +94,7 @@ void NHttpMsgHeader::update_flow() { (*(int64_t*)header_value_norm[HEAD_CONTENT_LENGTH].start > 0)) { // Regular body session_data->type_expected[source_id] = SEC_BODY; - session_data->octets_expected[source_id] = *(int64_t*)get_header_value_norm(HEAD_CONTENT_LENGTH).start; session_data->data_length[source_id] = *(int64_t*)get_header_value_norm(HEAD_CONTENT_LENGTH).start; - session_data->body_sections[source_id] = 0; session_data->body_octets[source_id] = 0; } else { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h index f07678902..d0c599038 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_header.h @@ -37,7 +37,8 @@ class NHttpMsgHeader: public NHttpMsgHeadShared { public: - NHttpMsgHeader(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgHeader(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void print_section(FILE *output); void gen_events(); void update_flow(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc index 1deb11840..796e19187 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc @@ -40,8 +40,9 @@ using namespace NHttpEnums; -NHttpMsgRequest::NHttpMsgRequest(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgStart(buffer, buf_size, session_data_, source_id_) +NHttpMsgRequest::NHttpMsgRequest(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgStart(buffer, buf_size, session_data_, source_id_, buf_owner) { transaction->set_request(this); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.h index 86edefe07..1ea16d90b 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.h @@ -41,7 +41,8 @@ class NHttpMsgRequest: public NHttpMsgStart { public: - NHttpMsgRequest(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgRequest(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); ~NHttpMsgRequest() { delete uri; }; void print_section(FILE *output); void gen_events(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc index 698055e05..bfa68a58a 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc @@ -41,7 +41,8 @@ using namespace NHttpEnums; -NHttpMsgSection::NHttpMsgSection(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : +NHttpMsgSection::NHttpMsgSection(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : msg_text(buf_size, buffer), session_data(session_data_), source_id(source_id_), @@ -51,7 +52,8 @@ NHttpMsgSection::NHttpMsgSection(const uint8_t *buffer, const uint16_t buf_size, infractions(session_data->infractions[source_id]), version_id(session_data->version_id[source_id]), method_id(session_data->method_id[source_id]), - status_code_num(session_data->status_code_num[source_id]) + status_code_num(session_data->status_code_num[source_id]), + delete_msg_on_destruct(buf_owner) {} // Return the number of octets before the first CRLF. Return length if CRLF not present. diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h index 98b9fd11f..06f63e52f 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h @@ -43,17 +43,20 @@ class NHttpMsgHeadShared; class NHttpMsgSection { public: - virtual ~NHttpMsgSection() { delete[] msg_text.start; }; - virtual void analyze() = 0; // Minimum necessary processing for every message - virtual void print_section(FILE *output) = 0; // Test tool prints all derived message parts - virtual void gen_events() = 0; // Converts collected information into required preprocessor events - virtual void update_flow() = 0; // Manages the splitter and communication between message sections - virtual void legacy_clients() = 0; // Populates the raw and normalized buffer interface used by old Snort + virtual ~NHttpMsgSection() { if (delete_msg_on_destruct) delete[] msg_text.start; }; + virtual void analyze() = 0; // Minimum necessary processing for every message + virtual void print_section(FILE *output) = 0; // Test tool prints all derived message parts + virtual void gen_events() = 0; // Converts collected information into required preprocessor events + virtual void update_flow() = 0; // Manages the splitter and communication between message sections + virtual void legacy_clients() = 0; // Populates the raw and normalized buffer interface used by old Snort + virtual NHttpEnums::ProcessResult worth_detection() // What should we do with this section after processing? + { return NHttpEnums::RES_INSPECT; }; NHttpEnums::MethodId get_method_id() { return method_id; }; protected: - NHttpMsgSection(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgSection(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); // Convenience methods static uint32_t find_crlf(const uint8_t* buffer, int32_t length, bool wrappable); @@ -65,12 +68,12 @@ protected: void legacy_header(bool use_trailer); void legacy_cookie(NHttpMsgHeadShared* header, NHttpEnums::SourceId source_id); - Field msg_text; + const Field msg_text; - NHttpFlowData* session_data; - NHttpEnums::SourceId source_id; + NHttpFlowData* const session_data; + const NHttpEnums::SourceId source_id; NHttpTransaction* transaction; - bool tcp_close; + const bool tcp_close; ScratchPad scratch_pad; // This is where all the derived values, extracted message parts, and normalized values are. @@ -81,6 +84,9 @@ protected: NHttpEnums::VersionId version_id; NHttpEnums::MethodId method_id; int32_t status_code_num; + +private: + const bool delete_msg_on_destruct; }; #endif diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc index 624916ecf..9af9703ae 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.cc @@ -83,3 +83,7 @@ void NHttpMsgStart::derive_version_id() { void NHttpMsgStart::gen_events() {} +ProcessResult NHttpMsgStart::worth_detection() { + return RES_INSPECT; +} + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.h index 4708a5d4c..08578d911 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_start.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_start.h @@ -40,10 +40,12 @@ class NHttpMsgStart: public NHttpMsgSection { public: void analyze(); void gen_events(); + NHttpEnums::ProcessResult worth_detection(); protected: - NHttpMsgStart(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_) : - NHttpMsgSection(buffer, buf_size, session_data_, source_id_) {}; + NHttpMsgStart(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner) : + NHttpMsgSection(buffer, buf_size, session_data_, source_id_, buf_owner) {}; virtual void parse_start_line() = 0; void derive_version_id(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_status.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_status.cc index 60a4ee2f1..d7f9ec3d0 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_status.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_status.cc @@ -39,8 +39,9 @@ using namespace NHttpEnums; -NHttpMsgStatus::NHttpMsgStatus(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgStart(buffer, buf_size, session_data_, source_id_) +NHttpMsgStatus::NHttpMsgStatus(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgStart(buffer, buf_size, session_data_, source_id_, buf_owner) { transaction->set_status(this); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_status.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_status.h index e05cfb3e1..50944b3eb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_status.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_status.h @@ -38,7 +38,8 @@ class NHttpMsgStatus: public NHttpMsgStart { public: - NHttpMsgStatus(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgStatus(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void analyze(); void print_section(FILE *output); void gen_events(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc index 39ee0ba01..a264585d1 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.cc @@ -37,8 +37,9 @@ using namespace NHttpEnums; -NHttpMsgTrailer::NHttpMsgTrailer(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, SourceId source_id_) : - NHttpMsgHeadShared(buffer, buf_size, session_data_, source_id_) +NHttpMsgTrailer::NHttpMsgTrailer(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + SourceId source_id_, bool buf_owner) : + NHttpMsgHeadShared(buffer, buf_size, session_data_, source_id_, buf_owner) { transaction->set_trailer(this, source_id); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h index 5b056f475..eb7871e56 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_trailer.h @@ -37,7 +37,8 @@ class NHttpMsgTrailer: public NHttpMsgHeadShared { public: - NHttpMsgTrailer(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, NHttpEnums::SourceId source_id_); + NHttpMsgTrailer(const uint8_t *buffer, const uint16_t buf_size, NHttpFlowData *session_data_, + NHttpEnums::SourceId source_id_, bool buf_owner); void print_section(FILE *output); void gen_events(); void update_flow(); diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc index 95ffae5d9..0398dd282 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc @@ -61,12 +61,9 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total unsigned len, uint32_t flags, unsigned& copied) { static THREAD_LOCAL StreamBuffer nhttp_buf; - if (flags & PKT_PDU_HEAD) { - section_buffer = new uint8_t[65536]; - } + NHttpFlowData* session_data = (NHttpFlowData*)flow->get_application_data(NHttpFlowData::nhttp_flow_id); SourceId source_id = to_server() ? SRC_CLIENT : SRC_SERVER; - copied = len; if (NHttpTestInput::test_input) { @@ -74,31 +71,95 @@ const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned /*total { return nullptr; } - uint8_t* buffer; - NHttpTestInput::test_input_source->reassemble(&buffer, len, source_id); + uint8_t* test_buffer; + NHttpTestInput::test_input_source->reassemble(&test_buffer, len, source_id, session_data); if (len == 0) { // There is no more test data - delete[] section_buffer; - section_buffer = nullptr; return nullptr; } - data = buffer; + data = test_buffer; offset = 0; } - memcpy(section_buffer+offset, data, len); + bool is_chunk_body = session_data->section_type[source_id] == SEC_CHUNKBODY; + + uint8_t*& chunk_buffer = session_data->chunk_buffer[source_id]; + int32_t& chunk_buffer_length = session_data->chunk_buffer_length[source_id]; + uint8_t*& buffer = !is_chunk_body ? session_data->section_buffer[source_id] : chunk_buffer; + int32_t& buffer_length = !is_chunk_body ? session_data->section_buffer_length[source_id] : chunk_buffer_length; + + if (buffer == nullptr) { + buffer = new uint8_t[65536]; + } + + memcpy(buffer + buffer_length + offset, data, len); if (flags & PKT_PDU_TAIL) { - my_inspector->process(section_buffer, offset + len, flow, source_id); - nhttp_buf.data = section_buffer; - nhttp_buf.length = offset + len; - section_buffer = nullptr; // the buffer is the responsibility of the inspector now - return &nhttp_buf; + ProcessResult send_to_detection; + if (!is_chunk_body) { + // start line/headers/body individual section processing with aggregation prior to being sent to detection + // only the last section added to the buffer goes to the inspector + send_to_detection = my_inspector->process(buffer + buffer_length, offset + len, flow, source_id, + buffer_length == 0); + } + else { + // Because of aggregation chunk body sections do not go to Inspector on schedule or in chronological order + // with respect to otherchunks. That means NHttpMsgChunkBody::update_flow() cannot do it design-intended + // job of updating type_expected in time for StreamSplitter to find the next chunk header. So we do it here. + session_data->type_expected[source_id] = SEC_CHUNKHEAD; + + // small chunks are aggregated before processing and are kept here until the buffer is full (paf_max) + // all the chunks in the buffer go to the inspector together + int32_t total_chunk_len = chunk_buffer_length + offset + len; + if (total_chunk_len < 16384) { + paf_max = 16384 - total_chunk_len; + chunk_buffer_length = total_chunk_len; + return nullptr; + } + else { + paf_max = 16384; + } + send_to_detection = my_inspector->process(chunk_buffer, total_chunk_len, flow, source_id, true); + } + + // Buffers are reset to nullptr without delete[] because NHttpMsgSection holds the pointer and is responsible + switch (send_to_detection) { + case RES_INSPECT: + nhttp_buf.data = buffer; + nhttp_buf.length = buffer_length + offset + len; + buffer = nullptr; + buffer_length = 0; + return &nhttp_buf; + case RES_IGNORE: + buffer = nullptr; + buffer_length = 0; + return nullptr; + case RES_AGGREGATE: + buffer_length += offset + len; + return nullptr; + case RES_FLUSHCHUNKS: + buffer = nullptr; + buffer_length = 0; + if (chunk_buffer != nullptr) { + // FIXIT-M these three variables about the buffered chunks need to be managed properly + session_data->section_type[source_id] = SEC_CHUNKBODY; + session_data->tcp_close[source_id] = false; + session_data->infractions[source_id] = 0; + + my_inspector->process(chunk_buffer, chunk_buffer_length, flow, source_id, true); + nhttp_buf.data = chunk_buffer; + nhttp_buf.length = chunk_buffer_length; + chunk_buffer = nullptr; + chunk_buffer_length = 0; + return &nhttp_buf; + } + return nullptr; + } } return nullptr; } StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t length, uint32_t, uint32_t* flush_offset) { - // When the system begins providing TCP connection close information this won't always be false. &&& + // When the system begins providing TCP connection close information this won't always be false. FIXIT-H bool tcp_close = false; // This is the session state information we share with HTTP Inspect and store with stream. A session is defined @@ -116,9 +177,13 @@ StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* dat bool need_break; uint8_t* test_data = nullptr; NHttpTestInput::test_input_source->scan(test_data, length, source_id, tcp_close, need_break); - if (length == 0) return StreamSplitter::FLUSH; + if (length == 0) { + return StreamSplitter::FLUSH; + } data = test_data; - if (need_break) flow->set_application_data(session_data = new NHttpFlowData); + if (need_break) { + flow->set_application_data(session_data = new NHttpFlowData); + } } switch (SectionType type = session_data->type_expected[source_id]) { @@ -139,12 +204,12 @@ StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* dat session_data->num_crlf[source_id] = 0; } - // Check start line for leading CRLF because some 1.0 implementations put extra blank lines between messages. - // We tolerate this by quietly ignoring them. Header/trailer may also have leading CRLF. That is completely - // normal and means there are no header/trailer lines. - if ((session_data->num_crlf[source_id] == 2) && (session_data->octets_seen[source_id] == 2) && (type != SEC_CHUNKHEAD)) { + // If the first two octets are CRLF then flush them separately. We are 1) DISCARDing CRLF some + // 1.0 implementation put following previous message, 2) DISCARDing CRLF between chunk and following + // chunk header, and 3) flushing normal empty header or trailer. + if ((session_data->num_crlf[source_id] == 2) && (session_data->octets_seen[source_id] == 2)) { prepare_flush(session_data, flush_offset, source_id, - ((type == SEC_REQUEST) || (type == SEC_STATUS)) ? SEC_DISCARD : type, + ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD)) ? SEC_DISCARD : type, tcp_close && (k == length-1), 0, k+1); return StreamSplitter::FLUSH; } @@ -180,9 +245,9 @@ StreamSplitter::Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* dat return StreamSplitter::FLUSH; case SEC_BODY: case SEC_CHUNKBODY: - paf_max = 16384; - if ((!tcp_close) || (length > session_data->octets_expected[source_id])) { - prepare_flush(session_data, flush_offset, source_id, type, false, 0, session_data->octets_expected[source_id]); + paf_max = 16384 - session_data->chunk_buffer_length[source_id]; + if ((!tcp_close) || (length > session_data->data_length[source_id])) { + prepare_flush(session_data, flush_offset, source_id, type, false, 0, session_data->data_length[source_id]); } else { // The TCP connection has closed and this is the possibly incomplete final section diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h index 9700f30c4..ee89d1936 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h @@ -31,6 +31,7 @@ #include "stream/stream_splitter.h" #include "nhttp_flow_data.h" +#include "nhttp_test_input.h" class NHttpInspect; @@ -38,12 +39,11 @@ class NHttpStreamSplitter : public StreamSplitter { public: NHttpStreamSplitter(bool is_client_to_server, NHttpInspect* my_inspector_) : StreamSplitter(is_client_to_server), my_inspector(my_inspector_) { }; - ~NHttpStreamSplitter() { delete[] section_buffer; }; Status scan(Flow* flow, const uint8_t* data, uint32_t length, uint32_t not_used, uint32_t* flush_offset); const StreamBuffer* reassemble(Flow* flow, unsigned total, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied); bool is_paf() { return true; }; - uint32_t max() { return paf_max; }; + uint32_t max() { return NHttpTestInput::test_input ? 16384 : paf_max; }; private: void prepare_flush(NHttpFlowData* session_data, uint32_t* flush_offset, NHttpEnums::SourceId source_id, NHttpEnums::SectionType section_type, bool tcp_close, uint64_t infractions, uint32_t num_octets); @@ -51,7 +51,6 @@ private: NHttpInspect* const my_inspector; - uint8_t *section_buffer = nullptr; uint32_t paf_max = 63780; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc index a85c6d9e0..ced08f912 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc @@ -140,14 +140,6 @@ void NHttpTestInput::scan(uint8_t*& data, uint32_t &length, SourceId &source_id, tcp_close = true; tcp_closed = true; } - else if ((command_length == strlen("bodyend")) && !memcmp(command_value, "bodyend", strlen("bodyend"))) { - term_bytes[0] = 'x'; - term_bytes[1] = 'y'; - } - else if ((command_length == strlen("chunkend")) && !memcmp(command_value, "chunkend", strlen("chunkend"))) { - term_bytes[0] = '\r'; - term_bytes[1] = '\n'; - } else if (command_length > 0) { // Look for a test number bool is_number = true; @@ -160,6 +152,10 @@ void NHttpTestInput::scan(uint8_t*& data, uint32_t &length, SourceId &source_id, test_number = test_number * 10 + (command_value[j] - '0'); } } + else { + // Bad command in test file + assert(0); + } } } else { @@ -227,7 +223,7 @@ void NHttpTestInput::flush(uint32_t length) { } -void NHttpTestInput::reassemble(uint8_t **buffer, unsigned &length, SourceId &source_id) { +void NHttpTestInput::reassemble(uint8_t **buffer, unsigned &length, SourceId &source_id, NHttpFlowData* session_data) { source_id = last_source_id; *buffer = msg_buf; @@ -236,25 +232,14 @@ void NHttpTestInput::reassemble(uint8_t **buffer, unsigned &length, SourceId &so length = flush_octets; } else { - // We need to generate additional data to fill out the body or chunk section - // We may come through here multiple times as we generate all the PAF max body sections needed for a single flush - length = (flush_octets <= 16384) ? flush_octets : 16384; + // We need to generate additional data to fill out the body or chunk section. We may come through here + // multiple times as we generate all the maximum size body sections needed for a single flush. + uint32_t paf_max = 16384 - session_data->chunk_buffer_length[source_id]; + length = (flush_octets <= paf_max) ? flush_octets : paf_max; for (uint32_t k = end_offset; k < length; k++) { msg_buf[k] = 'A' + k % 26; } - flush_octets -= length; - - if (flush_octets == 0) { - if (length-end_offset > 1) { - msg_buf[length-2] = term_bytes[0]; - } - msg_buf[length-1] = term_bytes[1]; - } - else if (flush_octets == 1) { - msg_buf[length-1] = term_bytes[0]; - } - end_offset = 0; } } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h index 911e55a3c..626eaed5d 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h @@ -29,13 +29,15 @@ #ifndef NHTTP_TEST_INPUT_H #define NHTTP_TEST_INPUT_H +#include "nhttp_flow_data.h" + class NHttpTestInput { public: NHttpTestInput(const char *fileName); ~NHttpTestInput(); void scan(uint8_t*& data, uint32_t &length, NHttpEnums::SourceId &source_id, bool &tcp_close, bool &need_break); void flush(uint32_t length); - void reassemble(uint8_t **buffer, unsigned &length, NHttpEnums::SourceId &source_id); + void reassemble(uint8_t **buffer, unsigned &length, NHttpEnums::SourceId &source_id, NHttpFlowData* session_data); static bool test_input; static NHttpTestInput *test_input_source; @@ -50,7 +52,6 @@ private: uint32_t end_offset = 0; // last read character in the buffer int64_t test_number = 0; // for numbering test output files NHttpEnums::SourceId last_source_id = NHttpEnums::SRC_CLIENT; // current direction of traffic flow. Toggled by commands in file. - uint8_t term_bytes[2] = { 'x', 'y' }; }; #endif diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt index 14e242c03..3ac973d90 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt @@ -15,7 +15,6 @@ # Command lines are left justified, lower case, with no whitespace: # @break resets HTTP Inspect data structures and begins a new test. Use it liberally to prevent unrelated tests from interfering with each other. # @request and @response set the message direction. Applies to subsequent sections until changed. -# @bodyend and @chunkend set the final two characters of a body or chunk to "xy" or "\r\n". # @ sets the test number and hence the test output file name. Applies to subsequent sections until changed. Don't reuse numbers. # # Escape sequences begin with '\'. They may be used within a paragraph or to begin a paragraph. @@ -32,8 +31,7 @@ # Whenever a segment contains insufficient data to make up a body or chunk, fill data will be generated to make up the difference based on the # Content-Length field or chunk header. Specifically flushing more data than in the current segment will trigger filling. The user should include at # least one character of body/chunk data either as part of the previous header segment or at the beginning of a new segment following the headers. -# All data bytes included in the file will be used followed by required fill data in the pattern ABC...XYZABC... The final two characters will be -# determined by bodyend "xy" or chunkend "\r\n". +# All data bytes included in the file will be used followed by required fill data in the pattern ABC...XYZABC... # *********************************************************************************************** @@ -484,21 +482,18 @@ HTTP/2.0 200 OK\r\nContent-type: text/plain\r\nTransfer-Encoding: gzip\r\nTransf # Valid Content-Length and body @7001 @break -@bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n I'm a message body. @7002 @break -@bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n 12345 @7003 @break -@bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n @@ -506,7 +501,6 @@ HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n @7004 @break -@bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n @@ -514,7 +508,6 @@ I'm a message body. @7005 @break -@bodyend @response HTTP/1.1 200 OK\r\n Transfer-Encoding: identity\r\n @@ -532,7 +525,6 @@ abcdefghijklmnopqrstuvwxyz @7006 @break -@bodyend @request POST /body/in/a/request/ HTTP/1.1\r\n Content-Length:16383\r\n @@ -542,7 +534,6 @@ Content-Length:16383\r\n @7007 @break -@bodyend @request POST /body/in/a/request/ HTTP/1.1\r\n Content-Length:16384\r\n @@ -551,7 +542,6 @@ Content-Length:16384\r\n @7008 @break -@bodyend @response HTTP/1.0 408 barely too big for one section\r\n Content-Length:16385\r\n @@ -561,7 +551,6 @@ Content-Length:16385\r\n @7009 @break -@bodyend @response HTTP/1.1 408 barely too big for two sections\r\n Content-Length:32772\r\n @@ -570,7 +559,6 @@ Content-Length:32772\r\n @7010 @break -@bodyend @response HTTP/1.1 408 more than eight sections\r\n Content-Length:133072\r\n @@ -597,7 +585,6 @@ HTTP/1.1 200 Silly gigantic length\r\nContent-type: \ttext/plain\t\r\nContent-LE # Remember chunk lengths are required to be specified in hex @9001 @break -@chunkend @response HTTP/1.1 208 Example with chunks\r\nTransfer-Encoding: chunked\r\n\r\n @@ -616,7 +603,6 @@ e6\r\nchunkoflength230 @9002 @break -@chunkend @request POST /request/with/chunks HTTP/1.1\r\n Content-Type: text/plain\r\n