From: Tom Peters Date: Thu, 14 Aug 2014 18:19:29 +0000 (-0400) Subject: mods to call process from splitter X-Git-Tag: 3.0.0-233~1428 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4c167d7614709a1a9b93cb7110bf3685516b331d;p=thirdparty%2Fsnort3.git mods to call process from splitter --- diff --git a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h index 379c20430..e1b383056 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_flow_data.h @@ -53,12 +53,15 @@ public: private: void halfReset(NHttpEnums::SourceId sourceId); + // StreamSplitter internal data + int64_t octetsSeen[2] = { 0, 0 }; + int numCrlf[2] = { 0, 0 }; + // StreamSplitter => Inspector (facts about the most recent message section) // 0 element refers to client request, 1 element refers to server response NHttpEnums::SectionType sectionType[2] = { NHttpEnums::SEC__NOTCOMPUTE, NHttpEnums::SEC__NOTCOMPUTE }; bool tcpClose[2] = { false, false }; uint64_t infractions[2] = { 0, 0 }; - uint64_t eventsGenerated[2] = { 0, 0 }; // Inspector => StreamSplitter (facts about the message section that is coming next) NHttpEnums::SectionType typeExpected[2] = { NHttpEnums::SEC_REQUEST, NHttpEnums::SEC_STATUS }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc index e0aea57cc..71ba0b983 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.cc @@ -117,12 +117,6 @@ void NHttpInspect::show(SnortConfig*) LogMessage("NHttpInspect\n"); } -void NHttpInspect::eval(Packet*) -{ - printf("eval()\n"); fflush(nullptr); /* &&& */ - return; -} - void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* const flow, SourceId sourceId) { NHttpFlowData* sessionData = (NHttpFlowData*)flow->get_application_data(NHttpFlowData::nhttp_flow_id); @@ -130,40 +124,18 @@ void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* cons NHttpMsgSection *msgSection = nullptr; - if (!NHttpTestInput::test_input) { - switch (sessionData->sectionType[sourceId]) { - case SEC_REQUEST: msgSection = new NHttpMsgRequest(data, dsize, sessionData, sourceId); break; - case SEC_STATUS: msgSection = new NHttpMsgStatus(data, dsize, sessionData, sourceId); break; - case SEC_HEADER: msgSection = new NHttpMsgHeader(data, dsize, sessionData, sourceId); break; - case SEC_BODY: msgSection = new NHttpMsgBody(data, dsize, sessionData, sourceId); break; - case SEC_CHUNKHEAD: msgSection = new NHttpMsgChunkHead(data, dsize, sessionData, sourceId); break; - case SEC_CHUNKBODY: msgSection = new NHttpMsgChunkBody(data, dsize, sessionData, sourceId); break; - case SEC_TRAILER: msgSection = new NHttpMsgTrailer(data, dsize, sessionData, sourceId); break; - case SEC_DISCARD: return; - default: assert(0); return; - } - } - else { - uint8_t *testBuffer; - uint16_t testLength; - if ((testLength = NHttpTestInput::testInput->toEval(&testBuffer, testNumber, sourceId)) > 0) { - switch (sessionData->sectionType[sourceId]) { - case SEC_REQUEST: msgSection = new NHttpMsgRequest(testBuffer, testLength, sessionData, sourceId); break; - case SEC_STATUS: msgSection = new NHttpMsgStatus(testBuffer, testLength, sessionData, sourceId); break; - case SEC_HEADER: msgSection = new NHttpMsgHeader(testBuffer, testLength, sessionData, sourceId); break; - case SEC_BODY: msgSection = new NHttpMsgBody(testBuffer, testLength, sessionData, sourceId); break; - case SEC_CHUNKHEAD: msgSection = new NHttpMsgChunkHead(testBuffer, testLength, sessionData, sourceId); break; - case SEC_CHUNKBODY: msgSection = new NHttpMsgChunkBody(testBuffer, testLength, sessionData, sourceId); break; - case SEC_TRAILER: msgSection = new NHttpMsgTrailer(testBuffer, testLength, sessionData, sourceId); break; - case SEC_DISCARD: return; - default: assert(0); return; - } - } - else { - printf("Zero length test data.\n"); - return; - } + switch (sessionData->sectionType[sourceId]) { + case SEC_REQUEST: msgSection = new NHttpMsgRequest(data, dsize, sessionData, sourceId); break; + case SEC_STATUS: msgSection = new NHttpMsgStatus(data, dsize, sessionData, sourceId); break; + case SEC_HEADER: msgSection = new NHttpMsgHeader(data, dsize, sessionData, sourceId); break; + case SEC_BODY: msgSection = new NHttpMsgBody(data, dsize, sessionData, sourceId); break; + case SEC_CHUNKHEAD: msgSection = new NHttpMsgChunkHead(data, dsize, sessionData, sourceId); break; + case SEC_CHUNKBODY: msgSection = new NHttpMsgChunkBody(data, dsize, sessionData, sourceId); break; + case SEC_TRAILER: msgSection = new NHttpMsgTrailer(data, dsize, sessionData, sourceId); break; + case SEC_DISCARD: delete[] data; return; + default: assert(0); delete[] data; return; } + msgSection->analyze(); msgSection->updateFlow(); msgSection->genEvents(); @@ -172,15 +144,15 @@ void NHttpInspect::process(const uint8_t* data, const uint16_t dsize, Flow* cons if (test_output) { if (!NHttpTestInput::test_input) msgSection->printSection(stdout); else { - if (testNumber != fileTestNumber) { + if (NHttpTestInput::testInput->getTestNumber() != fileTestNumber) { if (testOut) fclose (testOut); - fileTestNumber = testNumber; + fileTestNumber = NHttpTestInput::testInput->getTestNumber(); char fileName[100]; - snprintf(fileName, sizeof(fileName), "%s%" PRIi64 ".txt", testOutputPrefix, testNumber); + snprintf(fileName, sizeof(fileName), "%s%" PRIi64 ".txt", testOutputPrefix, fileTestNumber); if ((testOut = fopen(fileName, "w+")) == nullptr) throw std::runtime_error("Cannot open test output file"); } msgSection->printSection(testOut); - printf("Finished processing section from test %" PRIi64 "\n", testNumber); + printf("Finished processing section from test %" PRIi64 "\n", fileTestNumber); } } } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h index f1f869f9d..3f45487dc 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_inspect.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_inspect.h @@ -50,7 +50,7 @@ public: bool configure(SnortConfig*); int verify(SnortConfig*); void show(SnortConfig*); - void eval(Packet*); + void eval(Packet*) { return; }; bool enabled(); void tinit(); void tterm(); @@ -58,6 +58,7 @@ public: private: friend NHttpApi; + friend NHttpStreamSplitter; void process(const uint8_t* data, const uint16_t dsize, Flow* const flow, NHttpEnums::SourceId sourceId_); @@ -66,7 +67,6 @@ private: const char *testInputFile = "nhttp_test_msgs.txt"; const char *testOutputPrefix = "nhttpresults/testcase"; FILE *testOut = nullptr; - int64_t testNumber = 0; int64_t fileTestNumber = -1; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc index 86690c400..e0116a659 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.cc @@ -39,12 +39,10 @@ using namespace NHttpEnums; NHttpMsgSection::NHttpMsgSection(const uint8_t *buffer, const uint16_t bufSize, NHttpFlowData *sessionData_, SourceId sourceId_) : sessionData(sessionData_), sourceId(sourceId_), tcpClose(sessionData->tcpClose[sourceId]), scratchPad(2*bufSize+500), - infractions(sessionData->infractions[sourceId]), eventsGenerated(sessionData->eventsGenerated[sourceId]), - versionId(sessionData->versionId[sourceId]), methodId(sessionData->methodId[sourceId]), statusCodeNum(sessionData->statusCodeNum[sourceId]) + infractions(sessionData->infractions[sourceId]), versionId(sessionData->versionId[sourceId]), + methodId(sessionData->methodId[sourceId]), statusCodeNum(sessionData->statusCodeNum[sourceId]) { - rawBuf = new uint8_t[bufSize]; - memcpy(rawBuf, buffer, bufSize); - msgText.start = rawBuf; + msgText.start = buffer; msgText.length = bufSize; } diff --git a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h index 411710903..8fa989328 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_msg_section.h @@ -40,7 +40,7 @@ class NHttpMsgSection { public: - virtual ~NHttpMsgSection() { delete[] rawBuf; }; + virtual ~NHttpMsgSection() { delete[] msgText.start; }; virtual void analyze() = 0; // Minimum necessary processing for every message virtual void printSection(FILE *output) = 0; // Test tool prints all derived message parts virtual void genEvents() = 0; // Converts collected information into required preprocessor events @@ -56,9 +56,6 @@ protected: void printMessageWrapup(FILE *output) const; void createEvent(NHttpEnums::EventSid sid); - // The current strategy is to copy the entire raw message section into this object. Here it is. - uint8_t* rawBuf; - // This pseudonym for rawBuf isolates details of how the raw message is stored from everything else. Field msgText; NHttpFlowData* sessionData; @@ -70,7 +67,7 @@ protected: // These are all scalars, buffer pointers, and buffer sizes. The actual buffers are in message buffer (raw pieces) or the // scratchPad (normalized pieces). uint64_t infractions; - uint64_t eventsGenerated; + uint64_t eventsGenerated = 0; NHttpEnums::VersionId versionId; NHttpEnums::MethodId methodId; int32_t statusCodeNum; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc index 847da436c..53fd57f15 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.cc @@ -34,6 +34,7 @@ #include "nhttp_enum.h" #include "nhttp_test_input.h" #include "nhttp_stream_splitter.h" +#include "nhttp_inspect.h" using namespace NHttpEnums; @@ -43,30 +44,48 @@ void NHttpStreamSplitter::prepareFlush(NHttpFlowData* sessionData, uint32_t* flu sessionData->sectionType[sourceId] = sectionType; sessionData->tcpClose[sourceId] = tcpClose; sessionData->infractions[sourceId] = infractions; - sessionData->eventsGenerated[sourceId] = eventsGenerated; if (tcpClose) sessionData->typeExpected[sourceId] = SEC_CLOSED; if (!NHttpTestInput::test_input) *flushOffset = numOctets; - else NHttpTestInput::testInput->pafFlush(numOctets); - octetsSeen = 0; - numCrlf = 0; + else NHttpTestInput::testInput->flush(numOctets); + sessionData->octetsSeen[sourceId] = 0; + sessionData->numCrlf[sourceId] = 0; } -void NHttpStreamSplitter::createEvent(EventSid sid) { - SnortEventqAdd(NHTTP_GID, (uint32_t)sid); - eventsGenerated |= 1 << (sid-1); -} - -const StreamBuffer* NHttpStreamSplitter::reassemble(unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) { +const StreamBuffer* NHttpStreamSplitter::reassemble(Flow* flow, unsigned offset, const uint8_t* data, unsigned len, + uint32_t flags, unsigned& copied) +{ static THREAD_LOCAL StreamBuffer nhttp_buf; if (flags & PKT_PDU_HEAD) { sectionBuffer = new uint8_t[65536]; } - memcpy(sectionBuffer+offset, data, len); + + SourceId sourceId = (flags & PKT_FROM_CLIENT) ? SRC_CLIENT : SRC_SERVER; + copied = len; + + if (NHttpTestInput::test_input) { + if (!(flags & PKT_PDU_TAIL)) + { + return nullptr; + } + uint8_t* buffer; + NHttpTestInput::testInput->reassemble(&buffer, len, sourceId); + if (len == 0) { + // There is no more test data + delete[] sectionBuffer; + sectionBuffer = nullptr; + return nullptr; + } + data = buffer; + offset = 0; + } + + memcpy(sectionBuffer+offset, data, len); if (flags & PKT_PDU_TAIL) { - process(sectionBuffer, offset + len, p->flow, to_server() ? SRC_CLIENT : SRC_SERVER); + myInspector->process(sectionBuffer, offset + len, flow, sourceId); nhttp_buf.data = sectionBuffer; nhttp_buf.length = offset + len; + sectionBuffer = nullptr; // the buffer is the responsibility of the inspector now return &nhttp_buf; } return nullptr; @@ -87,7 +106,7 @@ PAF_Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t if (NHttpTestInput::test_input) { *flushOffset = length; bool needBreak; - NHttpTestInput::testInput->toPaf((uint8_t*&)data, length, sourceId, tcpClose, needBreak); + NHttpTestInput::testInput->scan((uint8_t*&)data, length, sourceId, tcpClose, needBreak); if (length == 0) return PAF_FLUSH; if (needBreak) flow->set_application_data(sessionData = new NHttpFlowData); } @@ -100,29 +119,29 @@ PAF_Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t case SEC_TRAILER: pafMax = 63780; for (uint32_t k = 0; k < length; k++) { - octetsSeen++; + sessionData->octetsSeen[sourceId]++; // Count the alternating and characters we have seen in a row - if (((data[k] == '\r') && (numCrlf%2 == 0)) || ((data[k] == '\n') && (numCrlf%2 == 1))) numCrlf++; - else numCrlf = 0; + if (((data[k] == '\r') && (sessionData->numCrlf[sourceId]%2 == 0)) || ((data[k] == '\n') && (sessionData->numCrlf[sourceId]%2 == 1))) sessionData->numCrlf[sourceId]++; + else sessionData->numCrlf[sourceId] = 0; // Check start line for leading CRLF because some 1.0 implementations put extra blank lines between messages. We tolerate this by quietly ignoring them. // Header/trailer may also have leading CRLF. That is completely normal and means there are no header/trailer lines. - if ((numCrlf == 2) && (octetsSeen == 2) && (type != SEC_CHUNKHEAD)) { + if ((sessionData->numCrlf[sourceId] == 2) && (sessionData->octetsSeen[sourceId] == 2) && (type != SEC_CHUNKHEAD)) { prepareFlush(sessionData, flushOffset, sourceId, ((type == SEC_REQUEST) || (type == SEC_STATUS)) ? SEC_DISCARD : type, tcpClose && (k == length-1), 0, k+1); return PAF_FLUSH; } // The start line and chunk header section always end with the first - else if ((numCrlf == 2) && ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD))) { + else if ((sessionData->numCrlf[sourceId] == 2) && ((type == SEC_REQUEST) || (type == SEC_STATUS) || (type == SEC_CHUNKHEAD))) { prepareFlush(sessionData, flushOffset, sourceId, type, tcpClose && (k == length-1), 0, k+1); return PAF_FLUSH; } // The header and trailer sections always end with the first double - else if (numCrlf == 4) { + else if (sessionData->numCrlf[sourceId] == 4) { prepareFlush(sessionData, flushOffset, sourceId, type, tcpClose && (k == length-1), 0, k+1); return PAF_FLUSH; } // We must do this to protect ourself from buffer overrun. - else if (octetsSeen >= 63780) { + else if (sessionData->octetsSeen[sourceId] >= 63780) { prepareFlush(sessionData, flushOffset, sourceId, type, tcpClose && (k == length-1), INF_HEADTOOLONG, k+1); return PAF_FLUSH; } @@ -130,7 +149,7 @@ PAF_Status NHttpStreamSplitter::scan (Flow* flow, const uint8_t* data, uint32_t // Incomplete headers wait patiently for more data if (!tcpClose) return PAF_SEARCH; // Discard the oddball case where the new "message" starts with - else if ((octetsSeen == 1) && (numCrlf == 1)) prepareFlush(sessionData, flushOffset, sourceId, SEC_DISCARD, true, 0, length); + else if ((sessionData->octetsSeen[sourceId] == 1) && (sessionData->numCrlf[sourceId] == 1)) prepareFlush(sessionData, flushOffset, sourceId, SEC_DISCARD, true, 0, length); // TCP connection close, flush the partial header else prepareFlush(sessionData, flushOffset, sourceId, type, true, INF_TRUNCATED, length); return PAF_FLUSH; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h index cd9be24ef..c7dfe46b3 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_stream_splitter.h @@ -36,11 +36,14 @@ class NHttpInspect; class NHttpStreamSplitter : public StreamSplitter { public: - NHttpStreamSplitter(bool isClientToServer, NHttpInspect* myInspector_) : StreamSplitter(isClientToServer), myInspector(myInspector_) {}; + NHttpStreamSplitter(bool isClientToServer, NHttpInspect* myInspector_) : StreamSplitter(isClientToServer), + myInspector(myInspector_) {}; + ~NHttpStreamSplitter() { delete[] sectionBuffer; }; PAF_Status scan(Flow* flow, const uint8_t* data, uint32_t length, uint32_t flags, uint32_t* flushOffset); + const StreamBuffer* reassemble(Flow* flow, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, + unsigned& copied); bool is_paf() { return true; }; uint32_t max() { return pafMax; }; - const StreamBuffer* reassemble(unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied); private: void prepareFlush(NHttpFlowData* sessionData, uint32_t* flushOffset, NHttpEnums::SourceId sourceId, NHttpEnums::SectionType sectionType, bool tcpClose, uint64_t infractions, uint32_t numOctets); @@ -49,9 +52,6 @@ private: NHttpInspect* const myInspector; uint8_t *sectionBuffer = nullptr; - uint64_t eventsGenerated = 0; - int64_t octetsSeen = 0; - int numCrlf = 0; uint32_t pafMax = 63780; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc index 70cbff4d3..e2c14789f 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.cc @@ -52,28 +52,29 @@ NHttpTestInput::~NHttpTestInput() { // Read from the test data file and present to PAF. // In the process we may need to skip comments, execute simple commands, and handle escape sequences. // The best way to understand this function is to read the comments at the top of the file of test cases. -void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, bool &tcpClose, bool &needBreak) { - // No new data presented to PAF while the last section is still being flushed. - if (flushed) { - length = 0; - return; - } - +void NHttpTestInput::scan(uint8_t*& data, uint32_t &length, SourceId &sourceId, bool &tcpClose, bool &needBreak) { sourceId = lastSourceId; tcpClose = false; needBreak = false; + // Need to create and inspect additional message section(s) from the previous flush before we read new stuff + if ((endOffset == 0) && (flushOctets > 0)) { + length = 0; + return; + } + if (justFlushed) { - // PAF just flushed. There may or may not be leftover data in our buffer. + // PAF just flushed and it has all been sent to inspection. There may or may not be leftover data from the + // last segment that was not flushed. justFlushed = false; data = msgBuf; - length = endOffset - flushOffset; // this is the leftover data + length = endOffset - flushOctets; // this is the leftover data previousOffset = 0; endOffset = length; if (length > 0) { // Must present unflushed leftovers to PAF again. // If we don't take this opportunity to left justify our data in the buffer we may "walk" to the right until we run out of buffer space - memmove(msgBuf, msgBuf+flushOffset, length); + memmove(msgBuf, msgBuf+flushOctets, length); tcpClose = tcpAlreadyClosed; return; } @@ -89,7 +90,7 @@ void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, // Now we need to move forward by reading more data from the file int newChar; - typedef enum { WAITING, COMMENT, COMMAND, SECTION, ESCAPE, HEXVAL, FILLNUM, BRIDGE } State; + typedef enum { WAITING, COMMENT, COMMAND, SECTION, ESCAPE, HEXVAL } State; State state = WAITING; bool ending; int commandLength; @@ -97,7 +98,6 @@ void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, char commandValue[MaxCommand]; uint8_t hexVal; int numDigits; - uint32_t fillLength; while ((newChar = getc(testDataFile)) != EOF) { switch (state) { @@ -176,24 +176,14 @@ void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, case 'n': state = SECTION; data[length++] = '\n'; break; case 'r': state = SECTION; data[length++] = '\r'; break; case 't': state = SECTION; data[length++] = '\t'; break; - case 'B': state = BRIDGE; break; - case 'C': endOffset = previousOffset + length; return; - case 'T': tcpClose = tcpAlreadyClosed = true; endOffset = previousOffset + length; return; case '#': state = SECTION; data[length++] = '#'; break; case '@': state = SECTION; data[length++] = '@'; break; case '\\': state = SECTION; data[length++] = '\\'; break; case 'x': case 'X': state = HEXVAL; hexVal = 0; numDigits = 0; break; - case '/': state = FILLNUM; fillLength = 0; break; default: assert(0); state = SECTION; break; } break; - case BRIDGE: - if (newChar != '\n') { - state = SECTION; - data[length++] = (uint8_t) newChar; - } - break; case HEXVAL: if ((newChar >= '0') && (newChar <= '9')) hexVal = hexVal * 16 + (newChar - '0'); else if ((newChar >= 'a') && (newChar <= 'f')) hexVal = hexVal * 16 + 10 + (newChar - 'a'); @@ -204,30 +194,6 @@ void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, state = SECTION; } break; - case FILLNUM: - if (newChar != '/') { - assert((newChar >= '0') && (newChar <= '9')); - fillLength = fillLength * 10 + (newChar - '0'); - assert(fillLength <= sizeof(msgBuf)); - break; - } - else { - bodyData = true; - // Add the specified number of fill characters to the buffer and cut. - // Simulates body data at the end of a header segment or the first segment containing body data - // Don't allow a buffer overrun. - if (previousOffset + length + fillLength > sizeof(msgBuf)) assert(0); - for (uint32_t k=0; k < fillLength; k++) { - data[length++] = 'x'; - } - endOffset = previousOffset + length; - return; - } - } - // If we have reached the configured maximum segment size automatically cut the data. - if (length >= mssLength) { - endOffset = previousOffset + length; - return; } // Don't allow a buffer overrun. if (previousOffset + length >= sizeof(msgBuf)) assert(0); @@ -237,59 +203,42 @@ void NHttpTestInput::toPaf(uint8_t*& data, uint32_t &length, SourceId &sourceId, return; } -void NHttpTestInput::pafFlush(uint32_t length) { - assert(!flushed); - flushed = true; - if (bodyData && (previousOffset + length >= endOffset)) { - fillOctets = length; - bodyData = false; - previousOffset = 0; - endOffset = 0; - flushOffset = 0; - } - else { - flushOffset = previousOffset + length; - } +void NHttpTestInput::flush(uint32_t length) { + flushOctets = previousOffset + length; + justFlushed = true; } -uint16_t NHttpTestInput::toEval(uint8_t **buffer, int64_t &testNumber_, SourceId &sourceId) { - if (!flushed) return 0; - testNumber_ = testNumber; +void NHttpTestInput::reassemble(uint8_t **buffer, unsigned &length, SourceId &sourceId) { sourceId = lastSourceId; *buffer = msgBuf; - if (fillOctets > 0) { - uint32_t fillOut = (fillOctets <= 16384) ? fillOctets : 16384; - for (uint32_t k = 0; k < fillOut; k++) { + + if (flushOctets <= endOffset) { + // All the data we need comes from the file + length = flushOctets; + } + else { + // We need to generate additional data to fill out the body or chunk section + // We may come through here multiple times as we generate all the PAF max body sections needed for a single flush + length = (flushOctets <= 16384) ? flushOctets : 16384; + for (uint32_t k = endOffset; k < length; k++) { msgBuf[k] = 'A' + k % 26; } - fillOctets -= fillOut; - if (fillOctets == 0) { - if (fillOut > 1) msgBuf[fillOut-2] = termBytes[0]; - msgBuf[fillOut-1] = termBytes[1]; - flushed = false; - justFlushed = true; + + flushOctets -= length; + + if (flushOctets == 0) { + if (length-endOffset > 1) { + msgBuf[length-2] = termBytes[0]; + } + msgBuf[length-1] = termBytes[1]; } - else if (fillOctets == 1) { - msgBuf[fillOut-1] = termBytes[0]; + else if (flushOctets == 1) { + msgBuf[length-1] = termBytes[0]; } - return (uint16_t)fillOut; + + endOffset = 0; } - flushed = false; - justFlushed = true; - return (uint16_t)flushOffset; } - - - - - - - - - - - - diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h index 834458559..4034eb08a 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_input.h +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_input.h @@ -33,26 +33,22 @@ class NHttpTestInput { public: NHttpTestInput(const char *fileName); ~NHttpTestInput(); - void toPaf(uint8_t*& data, uint32_t &length, NHttpEnums::SourceId &sourceId, bool &tcpClose, bool &needBreak); - void pafFlush(uint32_t length); - uint16_t toEval(uint8_t **buffer, int64_t &testNumber, NHttpEnums::SourceId &sourceId); + void scan(uint8_t*& data, uint32_t &length, NHttpEnums::SourceId &sourceId, bool &tcpClose, bool &needBreak); + void flush(uint32_t length); + void reassemble(uint8_t **buffer, unsigned &length, NHttpEnums::SourceId &sourceId); - // Hard for NHttpInspect and PAF to share these without making them "global". This is as good a place as any for them to live. static bool test_input; static NHttpTestInput *testInput; + int64_t getTestNumber() { return testNumber; }; private: FILE *testDataFile; uint8_t msgBuf[2 * NHttpEnums::MAXOCTETS]; - bool flushed = false; // verifies alternation between PAF section flushing and eval() section processing - bool justFlushed = true; // toPaf() needs to do special post-flush processing before it resumes reading the file + bool justFlushed = true; // all octets sent to inspection and must resume reading the file bool tcpAlreadyClosed = false; // so we can keep presenting a TCP close to PAF until all the remaining octets are consumed and flushed - uint32_t flushOffset = 0; // last character in buffer that has been flushed by PAF and must go to eval(). + uint32_t flushOctets = 0; // number of octets that have been flushed and must go to inspection uint32_t previousOffset = 0; // last character in the buffer shown to PAF but not flushed yet uint32_t endOffset = 0; // last read character in the buffer - bool bodyData = false; // pending output of fill data - uint32_t fillOctets = 0; // remaining fill data int64_t testNumber = 0; // for numbering test output files - uint32_t mssLength = 1460; // Maximum Segment Size. Needs to be enhanced to be set through a command. NHttpEnums::SourceId lastSourceId = NHttpEnums::SRC_CLIENT; // current direction of traffic flow. Toggled by commands in file. uint8_t termBytes[2] = { 'x', 'y' }; }; diff --git a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt index 5d11baca2..256b01856 100644 --- a/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt +++ b/src/service_inspectors/nhttp_inspect/nhttp_test_msgs.txt @@ -20,19 +20,14 @@ # \# - # # \@ - @ # \xnn or \Xnn - where nn is a two-digit hexadecimal number. Insert an arbitrary 8-bit number as the next character. a-f and A-F are both acceptable. -# \/nnn/ - where nnn is a variable-length decimal number. Insert nnn octets of fill data. The fill characters are 'A' - 'Z' with the final two bytes -# "xy" for a normal body and "\r\n" for a chunk. -# \B, \C, and \T - bridge, cut, and TCP close. See below. - -# The default procedure for separating data into segments for presentation to PAF is 1) whenever a paragraph ends (blank line) and 2) every MSS octets, -# where MSS is currently a source code constant set to 1460. The \C escape forces an immediate separation point even in the middle of a paragraph. It is -# useful for testing PAF where message segments often arrive in pieces. \B causes prevents the immediately following blank line(s) from causing a -# separation point. Data resumes with the next non-newline character as if there was no gap. This allows PAF to be tested with the end of a section and -# the beginning of the next section or message in the same packet. Commands and comments following \B are not allowed. Data must resume immediately -# following the blank lines. + +# Data is separated into segments for presentation to PAF whenever a paragraph ends (blank line). # -# \T functions as \C but causes the TCP connection closed indication to be set. It only makes sense at the end of a paragraph. \C\T simulates a data -# packet subsequently followed by a close with no more data. +# Whenever a segment contains insufficient data to make up a body or chunk, fill data will be generated to make up the difference based on the +# Content-Length field or chunk header. Specifically flushing more data than in the current segment will trigger filling. The user should include at +# least one character of body/chunk data either as part of the previous header segment or at the beginning of a new segment following the headers. +# All data bytes included in the file will be used followed by required fill data in the pattern ABC...XYZABC... The final two characters will be +# determined by bodyend "xy" or chunkend "\r\n". # *********************************************************************************************** @@ -486,14 +481,14 @@ HTTP/2.0 200 OK\r\nContent-type: text/plain\r\nTransfer-Encoding: gzip\r\nTransf @bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n -\/19/ +I'm a message body. @7002 @break @bodyend @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n -\/10/ +12345 @7003 @break @@ -501,7 +496,7 @@ HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n -\/10/ +12345678 @7004 @break @@ -509,7 +504,7 @@ HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n @response HTTP/1.1 200 OK\r\nContent-type: \ttext/plain\t\r\nContent-LENGTH: 19\r\n\r\n -\/19/ +I'm a message body. @7005 @break @@ -520,7 +515,14 @@ Transfer-Encoding: identity\r\n CoNtEnT-lEnGtH:16382\r\n Content-type: text/plain\r\n \r\n -\/800/ +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz +abcdefghijklmnopqrstuvwxyz @7006 @break @@ -530,7 +532,7 @@ POST /body/in/a/request/ HTTP/1.1\r\n Content-Length:16383\r\n \r\n -\/1300/ +* @7007 @break @@ -539,7 +541,7 @@ Content-Length:16383\r\n POST /body/in/a/request/ HTTP/1.1\r\n Content-Length:16384\r\n \r\n -\/1/ +* @7008 @break @@ -549,7 +551,7 @@ HTTP/1.0 408 barely too big for one section\r\n Content-Length:16385\r\n \r\n -\/1/ +* @7009 @break @@ -558,7 +560,7 @@ Content-Length:16385\r\n HTTP/1.1 408 barely too big for two sections\r\n Content-Length:32772\r\n \r\n -\/2/ +** @7010 @break @@ -568,7 +570,7 @@ HTTP/1.1 408 more than eight sections\r\n Content-Length:133072\r\n \r\n -\/2/ +******** # *********************************************************************************************** # Invalid Content-Length @@ -594,14 +596,13 @@ HTTP/1.1 200 Silly gigantic length\r\nContent-type: \ttext/plain\t\r\nContent-LE HTTP/1.1 208 Example with chunks\r\nTransfer-Encoding: chunked\r\n\r\n 64\r\n -\/100/ +1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\r\n 8A\r\n -\/60/ -e6\r\n +chunkoflength138 -\/1/ +e6\r\nchunkoflength230 0\r\n @@ -615,32 +616,34 @@ POST /request/with/chunks HTTP/1.1\r\n Content-Type: text/plain\r\n Transfer-Encoding: compress\r\n Via: 1.1 proxy3.company.com\r\n -tRaNsFeR-eNcO\CdInG: cHuNkEd \r\n +tRaNsFeR-eNcO + +dInG: cHuNkEd \r\n Accept: *\r\n \r\n 1C320\r\n -\/1400/ +chunk1 1F40; testing-the-extension-feature\r\n -\/2/ +chunk2 13FFE; \r\n -\/1460/ +chunk3 4000;\r\n -\/1460/ +chunk4 3FFF;\r\n -\/2/ +chunk5 3FFE\r\n -\/1454/ +chunk6 4001;\r\n -\/1/ +chunk7 00000;chunkextension=3\r\n @@ -681,6 +684,7 @@ Accept-Language: is\r\n @request + # *********************************************************************************************** # Alerts @14001