From: Tom Peters Date: Wed, 27 Aug 2014 15:26:47 +0000 (-0400) Subject: pipelining X-Git-Tag: 3.0.0-233~1420 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8adaa726aa1ba2fddcfaee164d1858639865892a;p=thirdparty%2Fsnort3.git pipelining --- diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc index 7e294abfa..4f6d7472c 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.cc @@ -43,6 +43,7 @@ NHttpFlowData::NHttpFlowData() : FlowData(nhttp_flow_id) { } NHttpFlowData::~NHttpFlowData() { delete transaction[SRC_CLIENT]; delete transaction[SRC_SERVER]; + delete_pipeline(); } void NHttpFlowData::half_reset(SourceId source_id) { @@ -61,4 +62,43 @@ void NHttpFlowData::half_reset(SourceId source_id) { chunk_octets[source_id] = STAT_NOTPRESENT; } +bool NHttpFlowData::add_to_pipeline(NHttpTransaction* latest) { + assert(!pipeline_overflow && !pipeline_underflow); + int new_back = (pipeline_back+1) % MAX_PIPELINE; + if (new_back == pipeline_front) { + pipeline_overflow = true; + return false; + } + pipeline[pipeline_back] = latest; + pipeline_back = new_back; + return true; +} + +NHttpTransaction* NHttpFlowData::take_from_pipeline() { + assert(!pipeline_underflow); + if (pipeline_back == pipeline_front) { + return nullptr; + } + int old_front = pipeline_front; + pipeline_front = (pipeline_front+1) % MAX_PIPELINE; + return pipeline[old_front]; +} + +void NHttpFlowData::delete_pipeline() { + for (int k=pipeline_front; k != pipeline_back; k = (k+1) % MAX_PIPELINE) { + delete pipeline[k]; + } +} + + + + + + + + + + + + diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h index ce73600a5..d0293eddb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h @@ -29,6 +29,7 @@ #ifndef NHTTP_FLOW_DATA_H #define NHTTP_FLOW_DATA_H +#include #include "stream/stream_api.h" class NHttpTransaction; @@ -84,8 +85,17 @@ private: int64_t chunk_sections[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of sections seen so far in the current chunk int64_t chunk_octets[2] = { NHttpEnums::STAT_NOTPRESENT, NHttpEnums::STAT_NOTPRESENT }; // number of user data octets seen so far in the current chunk including terminating CRLF - // Phase 1: separate transactions in each direction. Need to combine them &&& + // Transaction management including pipelining NHttpTransaction* transaction[2] = { nullptr, nullptr }; + static const int MAX_PIPELINE = 100; // requests seen - responses seen <= MAX_PIPELINE + NHttpTransaction* pipeline[MAX_PIPELINE]; + int pipeline_front = 0; + int pipeline_back = 0; + bool pipeline_overflow = false; + bool pipeline_underflow = false; + bool add_to_pipeline(NHttpTransaction* latest); + NHttpTransaction* take_from_pipeline(); + void delete_pipeline(); }; #endif diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc index 736175c75..55c869391 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_body.cc @@ -42,7 +42,7 @@ NHttpMsgBody::NHttpMsgBody(const uint8_t *buffer, const uint16_t buf_size, NHttp NHttpMsgSection(buffer, buf_size, session_data_, source_id_), data_length(session_data->data_length[source_id]), body_sections(session_data->body_sections[source_id]), body_octets(session_data->body_octets[source_id]) { - transaction->set_other(this); + transaction->set_body(this); } void NHttpMsgBody::analyze() { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc index a94c77790..3d93f53c7 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_body.cc @@ -42,7 +42,7 @@ NHttpMsgChunkBody::NHttpMsgChunkBody(const uint8_t *buffer, const uint16_t buf_s NHttpMsgBody(buffer, buf_size, session_data_, source_id_), /* num_chunks(session_data->num_chunks[source_id]), &&& */ chunk_sections(session_data->chunk_sections[source_id]), chunk_octets(session_data->chunk_octets[source_id]) { - transaction->set_other(this); + transaction->set_body(this); } void NHttpMsgChunkBody::analyze() { diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc index 71a7b27f0..1168370cb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_chunk_head.cc @@ -42,7 +42,7 @@ NHttpMsgChunkHead::NHttpMsgChunkHead(const uint8_t *buffer, const uint16_t buf_s NHttpMsgSection(buffer, buf_size, session_data_, source_id_), body_sections(session_data->body_sections[source_id]), num_chunks(session_data->num_chunks[source_id]) { - transaction->set_other(this); + transaction->set_body(this); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc index 02f8692d3..1deb11840 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_request.cc @@ -121,26 +121,28 @@ void NHttpMsgRequest::print_section(FILE *output) { NHttpMsgSection::print_message_title(output, "request line"); fprintf(output, "Version Id: %d\n", version_id); fprintf(output, "Method Id: %d\n", method_id); - uri->get_uri().print(output, "URI"); - if (uri->get_uri_type() != URI__NOSOURCE) fprintf(output, "URI Type: %d\n", uri->get_uri_type()); - uri->get_scheme().print(output, "Scheme"); - if (uri->get_scheme_id() != SCH__NOSOURCE) fprintf(output, "Scheme Id: %d\n", uri->get_scheme_id()); - uri->get_authority().print(output, "Authority"); - uri->get_host().print(output, "Host Name"); - uri->get_norm_host().print(output, "Normalized Host Name"); - uri->get_port().print(output, "Port"); - if (uri->get_port_value() != STAT_NOSOURCE) fprintf(output, "Port Value: %d\n", uri->get_port_value()); - uri->get_abs_path().print(output, "Absolute Path"); - uri->get_path().print(output, "Path"); - uri->get_norm_path().print(output, "Normalized Path"); - uri->get_query().print(output, "Query"); - uri->get_norm_query().print(output, "Normalized Query"); - uri->get_fragment().print(output, "Fragment"); - uri->get_norm_fragment().print(output, "Normalized Fragment"); - fprintf(output, "URI infractions: overall %" PRIx64 ", format %" PRIx64 ", scheme %" PRIx64 ", host %" PRIx64 ", port %" PRIx64 ", path %" - PRIx64 ", query %" PRIx64 ", fragment %" PRIx64 "\n", - uri->get_uri_infractions(), uri->get_format_infractions(), uri->get_scheme_infractions(), uri->get_host_infractions(), - uri->get_port_infractions(), uri->get_path_infractions(), uri->get_query_infractions(), uri->get_fragment_infractions()); + if (uri != nullptr) { + uri->get_uri().print(output, "URI"); + if (uri->get_uri_type() != URI__NOSOURCE) fprintf(output, "URI Type: %d\n", uri->get_uri_type()); + uri->get_scheme().print(output, "Scheme"); + if (uri->get_scheme_id() != SCH__NOSOURCE) fprintf(output, "Scheme Id: %d\n", uri->get_scheme_id()); + uri->get_authority().print(output, "Authority"); + uri->get_host().print(output, "Host Name"); + uri->get_norm_host().print(output, "Normalized Host Name"); + uri->get_port().print(output, "Port"); + if (uri->get_port_value() != STAT_NOSOURCE) fprintf(output, "Port Value: %d\n", uri->get_port_value()); + uri->get_abs_path().print(output, "Absolute Path"); + uri->get_path().print(output, "Path"); + uri->get_norm_path().print(output, "Normalized Path"); + uri->get_query().print(output, "Query"); + uri->get_norm_query().print(output, "Normalized Query"); + uri->get_fragment().print(output, "Fragment"); + uri->get_norm_fragment().print(output, "Normalized Fragment"); + fprintf(output, "URI infractions: overall %" PRIx64 ", format %" PRIx64 ", scheme %" PRIx64 ", host %" PRIx64 ", port %" PRIx64 ", path %" + PRIx64 ", query %" PRIx64 ", fragment %" PRIx64 "\n", + uri->get_uri_infractions(), uri->get_format_infractions(), uri->get_scheme_infractions(), uri->get_host_infractions(), + uri->get_port_infractions(), uri->get_path_infractions(), uri->get_query_infractions(), uri->get_fragment_infractions()); + } NHttpMsgSection::print_message_wrapup(output); } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt index 980797af0..14e242c03 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt @@ -781,8 +781,64 @@ GET /trigger/basic/normalized/URI/content/match/alert HTTP/1.1\r\n\r\n GET /%74%72%69%67%67%65%72/encoded/normalized/URI/content/match/alert HTTP/1.1\r\n\r\n +# *********************************************************************************************** +# Pipelining +@15001 +@break +@request + +GET /pipeline/test/1 HTTP/1.1\r\n +Host: www.pipeline1.com\r\n +\r\n +GET /pipeline/test/2 HTTP/1.1\r\n +Host: www.pipeline2.com\r\n +\r\n +GET /pipeline/test/3 HTTP/1.1\r\n +Host: www.pipeline3.com\r\n +\r\n +GET /pipeline/test/4 HTTP/1.1\r\n +Host: www.pipeline4.com\r\n +\r\n + +GET /pipeline/test/5 HTTP/1.1\r\n +Host: www.pipeline5.com\r\n +\r\n +GET /pipeline/test/6 HTTP/1.1\r\n +Host: www.pipeline6.com\r\n +\r\n + +@response +HTTP/1.1 400 Response to pipeline 1\r\n +Content-Length: 10000\r\n +\r\n +Data 1 from pipeline\r\n + +HTTP/1.1 400 Response to pipeline 2\r\n +Content-Length: 20000\r\n +\r\n +Data 2 from pipeline\r\n +HTTP/1.1 400 Response to pipeline 3\r\n +Content-Length: 30000\r\n +\r\n + +Data 3 from pipeline\r\n + +HTTP/1.1 400 Response to pipeline 4\r\n +Content-Length: 40000\r\n +\r\n +Data 4 from pipeline\r\n +HTTP/1.1 400 Response to pipeline 5\r\n +Content-Length: 50000\r\n +\r\n + +Data 5 from pipeline\r\n + +HTTP/1.1 400 Response to pipeline 6\r\n +Content-Length: 60000\r\n +\r\n +Data 6 from pipeline\r\n diff --git a/src/service_inspectors/nhttp_inspect/nhttp_transaction.cc b/src/service_inspectors/nhttp_inspect/nhttp_transaction.cc index 815e1d925..4d0f46bdd 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_transaction.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_transaction.cc @@ -43,35 +43,64 @@ using namespace NHttpEnums; NHttpTransaction::~NHttpTransaction() { delete request; delete status; - delete latest_other; - for (int k=0; k <= 1; k++) { - delete header[k]; - delete trailer[k]; - } + delete header[0]; + delete header[1]; + delete trailer[0]; + delete trailer[1]; + delete latest_body; } NHttpTransaction* NHttpTransaction::attach_my_transaction(NHttpFlowData* session_data, SourceId source_id) { - SectionType section_type = session_data->section_type[source_id]; + // This factory method: + // 1. garbage collects most recent body section which is no longer needed once another section arrives + // 2. creates new transactions for all request messages and orphaned response messages + // 3. associates requests and responses and supports pipelining + // 4. garbage collects unneeded transactions + // 5. returns the current transaction + + if (session_data->transaction[SRC_CLIENT] != nullptr) { + delete session_data->transaction[SRC_CLIENT]->latest_body; + session_data->transaction[SRC_CLIENT]->latest_body = nullptr; + } + if (session_data->transaction[SRC_SERVER] != nullptr) { + delete session_data->transaction[SRC_SERVER]->latest_body; + session_data->transaction[SRC_SERVER]->latest_body = nullptr; + } - // If this is a request section we replace the previous transaction with a new transaction - if (section_type == SEC_REQUEST) { - delete session_data->transaction[SRC_CLIENT]; + // Request section: put the old transaction in the pipeline and replace it with a new transaction. If the pipeline + // overflows or underflows we stop using it and just delete the old transaction. + if (session_data->section_type[source_id] == SEC_REQUEST) { + // When pipelining is not occurring the response should already have taken this tranaction and left nullptr. + if (session_data->transaction[SRC_CLIENT] != nullptr) { + if ((session_data->pipeline_overflow) || (session_data->pipeline_underflow)) { + delete session_data->transaction[SRC_CLIENT]; + } + else if (!session_data->add_to_pipeline(session_data->transaction[SRC_CLIENT])) { + // The pipeline is full and just overflowed. FIXIT-M we should alert and set infraction. + delete session_data->transaction[SRC_CLIENT]; + } + } session_data->transaction[SRC_CLIENT] = new NHttpTransaction; } - // If this is a status section we replace the previous transaction, taking the latest request transaction if possible - else if (section_type == SEC_STATUS) { + // Status section: delete the current transaction and get a new one from the pipeline. If the pipeline is empty + // check for a request-side transaction that just finished and take it. If there is no transaction available then + // declare an underflow and create a new transaction specifically for the response side. + else if (session_data->section_type[source_id] == SEC_STATUS) { delete session_data->transaction[SRC_SERVER]; - if ((session_data->type_expected[SRC_CLIENT]) && (session_data->transaction[SRC_CLIENT] != nullptr)) { - session_data->transaction[SRC_SERVER] = session_data->transaction[SRC_CLIENT]; - session_data->transaction[SRC_CLIENT] = nullptr; - } - else { + if (session_data->pipeline_underflow) { session_data->transaction[SRC_SERVER] = new NHttpTransaction; } - } - else { - delete session_data->transaction[source_id]->latest_other; - session_data->transaction[source_id]->latest_other = nullptr; + else if ((session_data->transaction[SRC_SERVER] = session_data->take_from_pipeline()) == nullptr) { + if ((session_data->transaction[SRC_CLIENT] != nullptr) && + (session_data->type_expected[SRC_CLIENT] == SEC_REQUEST)) { + session_data->transaction[SRC_SERVER] = session_data->transaction[SRC_CLIENT]; + session_data->transaction[SRC_CLIENT] = nullptr; + } + else { + session_data->pipeline_underflow = true; + session_data->transaction[SRC_SERVER] = new NHttpTransaction; + } + } } return session_data->transaction[source_id]; @@ -80,10 +109,3 @@ NHttpTransaction* NHttpTransaction::attach_my_transaction(NHttpFlowData* session - - - - - - - diff --git a/src/service_inspectors/nhttp_inspect/nhttp_transaction.h b/src/service_inspectors/nhttp_inspect/nhttp_transaction.h index edd437c8b..6ee4dbcbb 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_transaction.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_transaction.h @@ -55,7 +55,7 @@ public: NHttpMsgTrailer* get_trailer(NHttpEnums::SourceId source_id) const { return trailer[source_id]; }; void set_trailer(NHttpMsgTrailer* trailer_, NHttpEnums::SourceId source_id) { trailer[source_id] = trailer_; }; - void set_other(NHttpMsgSection* latest_other_) { latest_other = latest_other_; }; + void set_body(NHttpMsgSection* latest_body_) { latest_body = latest_body_; }; private: NHttpTransaction() = default; @@ -64,7 +64,7 @@ private: NHttpMsgStatus* status = nullptr; NHttpMsgHeader* header[2] = { nullptr, nullptr }; NHttpMsgTrailer* trailer[2] = { nullptr, nullptr }; - NHttpMsgSection* latest_other = nullptr; + NHttpMsgSection* latest_body = nullptr; }; #endif