From: Mike Stepanek (mstepane) Date: Tue, 28 Jan 2020 17:13:01 +0000 (+0000) Subject: Merge pull request #1959 in SNORT/snort3 from ~KATHARVE/snort3:h2i_test_tool to master X-Git-Tag: 3.0.0-268~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4478b2fc58ae13793a001587a4308f77d024ac1e;p=thirdparty%2Fsnort3.git Merge pull request #1959 in SNORT/snort3 from ~KATHARVE/snort3:h2i_test_tool to master Squashed commit of the following: commit 403392e41d8058a6ffa1dc1adcdabe18665c5450 Author: Katura Harvey Date: Tue Jan 14 13:30:58 2020 -0500 http_inspect: update test tool to support the HTTP/2 macros and new insert command --- diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt index e15e4e6a4..a9a19c1f9 100644 --- a/src/service_inspectors/http_inspect/dev_notes.txt +++ b/src/service_inspectors/http_inspect/dev_notes.txt @@ -173,8 +173,9 @@ Each paragraph represents a TCP segment. The splitter can be tested by putting m the same paragraph (splitter must split) or continuing a section in the next paragraph (splitter must search and reassemble). -Lines beginning with # are comments. Lines beginning with @ are commands. This does not apply to -lines in the middle of a paragraph. +Lines beginning with # are comments. Lines beginning with @ are commands. These do not apply to +lines in the middle of a paragraph. Lines that begin with $ are insert commands - a special class of +commands that may be used within a paragraph the insert data into the message buffer. Commands: @break resets HTTP Inspect data structures and begins a new test. Use it liberally to prevent @@ -182,20 +183,31 @@ Commands: @tcpclose simulates a half-duplex TCP close. @request and @response set the message direction. Applies to subsequent paragraphs until changed. The initial direction is always request and the break command resets the direction to request. - @fill create a paragraph consisting of octets of auto-fill data - ABCDEFGHIJABC .... @partial causes a partial flush, simulating a retransmission of a detained packet @fileset specifies a file from which the tool will read data into the message buffer. This may be used to include a zipped or other binary file into a message body. Data is read beginning at the start of the file. The file is closed automatically whenever a new file is set or there is a break command. - @fileread read the specified number of bytes from the included file into the - message buffer. Each read corresponds to one TCP section. @fileskip skips over the specified number of bytes in the included file. This must be a positive number. To move backward do a new fileset and skip forward from the beginning. @ sets the test number and hence the test output file name. Applies to subsequent sections until changed. Don't reuse numbers. +Insert commands: + $fill create a paragraph consisting of octets of auto-fill data + ABCDEFGHIJABC .... + $fileread read the specified number of bytes from the included file into the + message buffer. Each read corresponds to one TCP section. + $h2preface creates the HTTP/2 connection preface "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + $h2frameheader generates an HTTP/2 frame header. + The frame type may be the frame type name in all lowercase or the numeric frame type code: + (data|headers|priority|rst_stream|settings|push_promise|ping|goaway|window_update| + continuation|{0:9}) + The frame length is the length of the frame payload, may be in decimal or test tool hex value + (\xnn, see below under escape sequence for more details) + The frame flags are represented as a single test tool hex byte (\xnn) + The stream id is optional. If provided it must be a decimal number. If not included it defaults + to 0. Escape sequences begin with '\'. They may be used within a paragraph or to begin a paragraph. \r - carriage return @@ -204,6 +216,7 @@ Escape sequences begin with '\'. They may be used within a paragraph or to begin \\ - backslash \# - # \@ - @ + \$ - $ \xnn or \Xnn - where nn is a two-digit hexadecimal number. Insert an arbitrary 8-bit number as the next character. a-f and A-F are both acceptable. diff --git a/src/service_inspectors/http_inspect/http_test_input.cc b/src/service_inspectors/http_inspect/http_test_input.cc index d7a592bf4..d46ca06fd 100644 --- a/src/service_inspectors/http_inspect/http_test_input.cc +++ b/src/service_inspectors/http_inspect/http_test_input.cc @@ -46,6 +46,41 @@ static unsigned convert_num_octets(const char buffer[], unsigned length) return amount; } +static void parse_next_hex_half_byte(const char new_char, uint8_t& hex_val) +{ + if ((new_char >= '0') && (new_char <= '9')) + hex_val = hex_val * 16 + (new_char - '0'); + else if ((new_char >= 'a') && (new_char <= 'f')) + hex_val = hex_val * 16 + 10 + (new_char - 'a'); + else if ((new_char >= 'A') && (new_char <= 'F')) + hex_val = hex_val * 16 + 10 + (new_char - 'A'); + else + assert(false); +} + +static uint8_t get_hex_byte(const char buffer[]) +{ + unsigned offset = 0; + assert(*buffer == '\\'); + offset++; + assert((*(buffer + offset) == 'X') or (*(buffer + offset) == 'x')); + offset++; + uint8_t hex_val = 0; + parse_next_hex_half_byte (*(buffer + offset++), hex_val); + parse_next_hex_half_byte (*(buffer + offset++), hex_val); + return hex_val; +} + +static bool is_number(const char buffer[], const unsigned length) +{ + for (unsigned k = 0; k < length; k++) + { + if (buffer[k] < '0' || buffer[k] > '9') + return false; + } + return true; +} + HttpTestInput::HttpTestInput(const char* file_name) { if ((test_data_file = fopen(file_name, "r")) == nullptr) @@ -136,7 +171,7 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u // Now we need to move forward by reading more data from the file int new_char; - enum State { WAITING, COMMENT, COMMAND, PARAGRAPH, ESCAPE, HEXVAL }; + enum State { WAITING, COMMENT, COMMAND, PARAGRAPH, ESCAPE, HEXVAL, INSERT}; State state = WAITING; bool ending = false; unsigned command_length = 0; @@ -164,12 +199,23 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u state = ESCAPE; ending = false; } + else if (new_char == '$') + { + state = INSERT; + command_length = 0; + } else if (new_char != '\n') { state = PARAGRAPH; ending = false; msg_buf[last_source_id][end_offset[last_source_id]++] = (uint8_t)new_char; } + else if (ending) + { + // An insert command was not followed by regular paragraph data + length = end_offset[last_source_id] - previous_offset[last_source_id]; + return; + } break; case COMMENT: if (new_char == '\n') @@ -210,20 +256,6 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u length = 0; return; } - else if ((command_length > strlen("fill")) && !memcmp(command_value, "fill", - strlen("fill"))) - { - const unsigned amount = convert_num_octets(command_value + strlen("fill"), - command_length - strlen("fill")); - assert((amount > 0) && (amount <= MAX_OCTETS)); - for (unsigned k = 0; k < amount; k++) - { - // auto-fill ABCDEFGHIJABCD ... - msg_buf[last_source_id][end_offset[last_source_id]++] = 'A' + k%10; - } - length = end_offset[last_source_id] - previous_offset[last_source_id]; - return; - } else if ((command_length == strlen("partial")) && !memcmp(command_value, "partial", strlen("partial"))) { @@ -249,6 +281,67 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u if ((include_file[last_source_id] = fopen(include_file_name, "r")) == nullptr) throw std::runtime_error("Cannot open test file to be included"); } + else if ((command_length > strlen("fileskip")) && !memcmp(command_value, + "fileskip", strlen("fileskip"))) + { + // Skip the specified number of octets from the included file + const unsigned amount = convert_num_octets(command_value + strlen("fileskip"), + command_length - strlen("fileskip")); + assert(amount > 0); + for (unsigned k=0; k < amount; k++) + { + getc(include_file[last_source_id]); + } + } + else if (command_length > 0) + { + // Look for a test number + if (is_number(command_value, command_length)) + { + int64_t test_number = 0; + for (unsigned j=0; j < command_length; j++) + { + test_number = test_number * 10 + (command_value[j] - '0'); + } + HttpTestManager::update_test_number(test_number); + } + else + { + // Bad command in test file + assert(false); + } + } + } + else + { + if (command_length < max_command) + { + command_value[command_length++] = new_char; + } + else + { + assert(false); + } + } + break; + case INSERT: + if (new_char == '\n') + { + state = WAITING; + ending = true; + if ((command_length > strlen("fill")) && !memcmp(command_value, "fill", + strlen("fill"))) + { + const unsigned amount = convert_num_octets(command_value + strlen("fill"), + command_length - strlen("fill")); + assert((amount > 0) && (amount <= MAX_OCTETS and + (amount < sizeof(msg_buf[last_source_id]) - end_offset[last_source_id]))); + for (unsigned k = 0; k < amount; k++) + { + // auto-fill ABCDEFGHIJABCD ... + msg_buf[last_source_id][end_offset[last_source_id]++] = 'A' + k%10; + } + } else if ((command_length > strlen("fileread")) && !memcmp(command_value, "fileread", strlen("fileread"))) { @@ -256,37 +349,31 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u // buffer and return the resulting segment const unsigned amount = convert_num_octets(command_value + strlen("fileread"), command_length - strlen("fileread")); - assert((amount > 0) && (amount <= MAX_OCTETS)); + assert((amount > 0) && (amount <= MAX_OCTETS and + (amount < sizeof(msg_buf[last_source_id]) - end_offset[last_source_id]))); for (unsigned k=0; k < amount; k++) { const int new_octet = getc(include_file[last_source_id]); assert(new_octet != EOF); msg_buf[last_source_id][end_offset[last_source_id]++] = new_octet; } - length = end_offset[last_source_id] - previous_offset[last_source_id]; - return; } - else if ((command_length > strlen("fileskip")) && !memcmp(command_value, - "fileskip", strlen("fileskip"))) + else if ((command_length > strlen("h2frameheader")) && !memcmp(command_value, + "h2frameheader", strlen("h2frameheader"))) { - // Skip the specified number of octets from the included file - const unsigned amount = convert_num_octets(command_value + strlen("fileskip"), - command_length - strlen("fileskip")); - assert(amount > 0); - for (unsigned k=0; k < amount; k++) - { - getc(include_file[last_source_id]); - } + generate_h2_frame_header(command_value, command_length); + } + else if ((command_length == strlen("h2preface")) && !memcmp(command_value, + "h2preface", strlen("h2preface"))) + { + char preface[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + memcpy(msg_buf[last_source_id] + end_offset[last_source_id], preface, sizeof(preface) - 1); + end_offset[last_source_id] += sizeof(preface) - 1; } else if (command_length > 0) { // Look for a test number - bool is_number = true; - for (unsigned k=0; (k < command_length) && is_number; k++) - { - is_number = (command_value[k] >= '0') && (command_value[k] <= '9'); - } - if (is_number) + if (is_number(command_value, command_length)) { int64_t test_number = 0; for (unsigned j=0; j < command_length; j++) @@ -333,6 +420,13 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u return; } } + else if (ending and new_char == '$') + { + // only look for insert commands at the start of a line + state = INSERT; + command_length = 0; + ending = false; + } else { ending = false; @@ -355,16 +449,11 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u msg_buf[last_source_id][end_offset[last_source_id]++] = '\t'; break; case '#': - state = PARAGRAPH; - msg_buf[last_source_id][end_offset[last_source_id]++] = '#'; - break; case '@': - state = PARAGRAPH; - msg_buf[last_source_id][end_offset[last_source_id]++] = '@'; - break; + case '$': case '\\': state = PARAGRAPH; - msg_buf[last_source_id][end_offset[last_source_id]++] = '\\'; + msg_buf[last_source_id][end_offset[last_source_id]++] = new_char; break; case 'x': case 'X': @@ -379,14 +468,7 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u } break; case HEXVAL: - if ((new_char >= '0') && (new_char <= '9')) - hex_val = hex_val * 16 + (new_char - '0'); - else if ((new_char >= 'a') && (new_char <= 'f')) - hex_val = hex_val * 16 + 10 + (new_char - 'a'); - else if ((new_char >= 'A') && (new_char <= 'F')) - hex_val = hex_val * 16 + 10 + (new_char - 'A'); - else - assert(false); + parse_next_hex_half_byte(new_char, hex_val); if (++num_digits == 2) { msg_buf[last_source_id][end_offset[last_source_id]++] = hex_val; @@ -447,6 +529,134 @@ void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, SourceId sour flushed = false; } +static uint8_t parse_frame_type(const char buffer[], const unsigned bytes_remaining, + unsigned& bytes_consumed) +{ + uint8_t frame_type = 0; + bytes_consumed = 0; + for (; bytes_consumed < bytes_remaining and buffer[bytes_consumed] == ' '; bytes_consumed++); + unsigned length = 0; + for (; (bytes_consumed + length < bytes_remaining) and (buffer[bytes_consumed + length] != ' '); + length++); + + static const char* frame_names[10] = + { "data", "headers", "priority", "rst_stream", "settings", "push_promise", "ping", "goaway", + "window_update", "continuation" }; + for (int i = 0; i < 10; i ++) + { + if (length == strlen(frame_names[i]) && !memcmp(buffer + bytes_consumed, frame_names[i], + strlen(frame_names[i]))) + { + frame_type = i; + bytes_consumed += length; + return frame_type; + } + } + if (is_number(buffer + bytes_consumed, length)) + frame_type = convert_num_octets(buffer + bytes_consumed, length); + else + assert(false); + + bytes_consumed += length; + return frame_type; +} + + +// Can be decimal or hex value. The hex value is represented as a series of 4-character hex bytes +// The hex value must not be more than 24-bits +static uint32_t get_frame_length(const char buffer[], const unsigned bytes_remaining, + unsigned& bytes_consumed) +{ + bytes_consumed = 0; + uint32_t frame_length = 0; + for (; bytes_consumed < bytes_remaining and buffer[bytes_consumed] == ' '; bytes_consumed++); + unsigned length = 0; + for (; (bytes_consumed + length < bytes_remaining) and (buffer[bytes_consumed + length] != ' '); + length++); + if (is_number(buffer + bytes_consumed, length)) + { + frame_length = convert_num_octets(buffer + bytes_consumed, length); + bytes_consumed += length; + } + else + { + assert(length >=3 and length <= 12 and length % 4 == 0); + unsigned end = bytes_consumed + length; + while (bytes_consumed < end) + { + frame_length <<= 8; + frame_length += get_hex_byte(buffer + bytes_consumed); + bytes_consumed += 4; + } + } + return frame_length; +} + +// Hex value represented as \xnn -- always 4 characters long +static uint8_t get_frame_flags(const char buffer[], const unsigned bytes_remaining, + unsigned& bytes_consumed) +{ + bytes_consumed = 0; + for (; bytes_consumed < bytes_remaining and buffer[bytes_consumed] == ' '; bytes_consumed++); + assert(bytes_remaining >= 4); + uint8_t frame_flags = get_hex_byte(buffer + bytes_consumed); + bytes_consumed += 4; + return frame_flags; +} + +// Check for optional stream_id in input. Default to stream 0 if not included +static uint32_t get_frame_stream_id(const char buffer[], const int bytes_remaining) +{ + int offset = 0; + for (; offset < bytes_remaining and buffer[offset] == ' '; offset++); + assert (bytes_remaining - offset >= 0); + int length = 0; + for (; (offset + length < bytes_remaining) and (buffer[offset + length] != ' '); length++); + if (length > 0) + { + if (is_number(buffer + offset, length)) + return convert_num_octets(buffer + offset, length); + else + assert(false); + } + return 0; +} + +void HttpTestInput::generate_h2_frame_header(const char command_value[], const unsigned command_length) +{ + unsigned offset = strlen("h2frameheader"); + unsigned bytes_consumed = 0; + uint8_t frame_type = 0; + uint8_t frame_flags = 0; + uint32_t frame_length = 0; + uint64_t stream_id = 0; + + // get the frame type + frame_type = parse_frame_type(command_value + offset, command_length - offset, bytes_consumed); + offset += bytes_consumed; + + frame_length = get_frame_length(command_value + offset, command_length - offset, bytes_consumed); + offset += bytes_consumed; + + assert (offset < command_length); + frame_flags = get_frame_flags(command_value + offset, command_length - offset, bytes_consumed); + offset += bytes_consumed; + + stream_id = get_frame_stream_id(command_value + offset, command_length - offset); + + // write the frame header + assert (!((frame_length >> (8*3)) & 0xFF)); + msg_buf[last_source_id][end_offset[last_source_id]++] = (frame_length >> 16) & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = (frame_length >> 8) & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = frame_length & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = frame_type; + msg_buf[last_source_id][end_offset[last_source_id]++] = frame_flags; + msg_buf[last_source_id][end_offset[last_source_id]++] = (stream_id >> 24) & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = (stream_id >> 16) & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = (stream_id >> 8) & 0xFF; + msg_buf[last_source_id][end_offset[last_source_id]++] = stream_id & 0xFF; +} + bool HttpTestInput::finish() { if (tcp_closed) diff --git a/src/service_inspectors/http_inspect/http_test_input.h b/src/service_inspectors/http_inspect/http_test_input.h index b626cab31..75da8b6a8 100644 --- a/src/service_inspectors/http_inspect/http_test_input.h +++ b/src/service_inspectors/http_inspect/http_test_input.h @@ -76,6 +76,8 @@ private: // number of characters in the buffer uint32_t end_offset[2] = { 0, 0 }; + void generate_h2_frame_header(const char command_value[], const unsigned command_length); + void reset(); };